generate_dataset_flexible_unipen.py

import dtw
import time
import numpy as np
import input_data_crossvalid as input_data
import network_settings as ns
import math
import csv
import os
import sys
from utils.proto_select import selector_selector, random_selection, center_selection, k_centers_selection, border_selection, spanning_selection


def get_dtwfeatures(proto_data, proto_number, local_sample):
    local_sample_length = np.shape(local_sample)[0]
    features = np.zeros((local_sample_length, proto_number))
    for prototype in range(proto_number):
        local_proto = proto_data[prototype]
        output, cost, DTW, path = dtw.dtw(local_proto, local_sample, extended=True)

        for f in range(local_sample_length):
            features[f, prototype] = cost[path[0][f]][path[1][f]]
    return features

def read_dtw_matrix(version, fold):
#    if not os.path.exists(os.path.join("data", "fold"+str(fold)+"-"+version+"-dtw-matrix.txt")):
#        exit("Please run cross_dtw_full.py first")
#    return np.genfromtxt(os.path.join("data", "fold"+str(fold)+"-"+version+"-dtw-matrix.txt"), delimiter=' ')
    if not os.path.exists(os.path.join("data", "all-"+version+"-dtw-matrix.txt")):
        exit("Please run cross_dtw_full.py first")
    full = np.genfromtxt(os.path.join("data", "all-"+version+"-dtw-matrix.txt"), delimiter=' ')
    number = full.shape[0]
    indices = np.arange(number)
    test_ratio = 0.1
    test_start = fold * int(test_ratio * float(number))
    test_end = (fold+1) * int(test_ratio * float(number))
    testset = indices[test_start:test_end]
    testset[::-1].sort()
    for pl in testset:
        indices = np.delete(indices, pl, 0)
    trainset = indices
    ret = full[trainset]
    return ret[:,trainset]



if __name__ == "__main__":
    if len(sys.argv) < 6:
        print("Error, Syntax: {0} [version] [prototype selection] [classwise/independent] [prototype number] [fold]".format(sys.argv[0]))
        exit()
    version = sys.argv[1]
    selection = sys.argv[2]
    classwise = sys.argv[3]
    proto_number = int(sys.argv[4])
    fold = int(sys.argv[5])

    print("Starting: {}".format(version))

    # load settings
    ns.load_settings_raw(version, "1d", 50, 2, fold)
    full_data_file = os.path.join("data", version + "-re-data.txt")
    full_label_file = os.path.join("data", version + "-re-labels.txt")
    # load data
    data_sets = input_data.read_data_sets(full_data_file, full_label_file, ns.IMAGE_SHAPE, test_ratio=0.1,
                                          validation_ratio=0.0, pickle=False, boring=False, fold=fold)

    no_classes = ns.NUM_CLASSES
    # print(proto_number)

    train_data = (data_sets.train.images.reshape((-1, 50, 2)) + 1.) * (
            127.5 / 127.)  # this input_data assumes images
    train_labels = data_sets.train.labels

    train_number = np.shape(train_labels)[0]

    test_data = (data_sets.test.images.reshape((-1, 50, 2)) + 1.) * (127.5 / 127.)  # this input_data assumes images
    test_labels = data_sets.test.labels
    test_number = np.shape(test_labels)[0]

    distances = [] if selection == "random" else read_dtw_matrix(version, fold)
    print(np.shape(train_labels))
    print(np.shape(distances))

    if classwise == "classwise":
        proto_loc = np.zeros(0, dtype=np.int32)
        proto_factor = int(proto_number / no_classes)
        for c in range(no_classes):
            cw = np.where(train_labels == c)[0]
            if selection == "random":
                cw_distances = []
            else:
                cw_distances = distances[cw]
                cw_distances = cw_distances[:,cw]
            cw_proto = selector_selector(selection, proto_factor, cw_distances)
            proto_loc = np.append(proto_loc, cw[cw_proto])
    else:
        proto_loc = selector_selector(selection, proto_number, distances)

    proto_data = train_data[proto_loc]
    print(proto_loc)
    #exit()
    #print("Selection Done.")

    # sorts the prototypes so deletion happens in reverse order and doesn't interfere with indices
    proto_loc[::-1].sort()

    # remove prototypes from training data
    for pl in proto_loc:
        train_data = np.delete(train_data, pl, 0)
        train_labels = np.delete(train_labels, pl, 0)

    # start generation
    test_label_fileloc = os.path.join("data", "fold{}-test-label-{}-{}-{}-{}.txt".format(fold, version, selection, classwise, proto_number))
    test_raw_fileloc = os.path.join("data", "fold{}-raw-test-data-{}-{}-{}-{}.txt".format(fold, version, selection, classwise, proto_number))
    test_dtw_fileloc = os.path.join("data", "fold{}-dtw_features-test-data-{}-{}-{}-{}.txt".format(fold, version, selection, classwise, proto_number))
    test_combined_fileloc = os.path.join("data", "fold{}-dtw_features-plus-raw-test-data-{}-{}-{}-{}.txt".format(fold, version, selection, classwise, proto_number))
    train_label_fileloc = os.path.join("data", "fold{}-train-label-{}-{}-{}-{}.txt".format(fold, version, selection, classwise, proto_number))
    train_raw_fileloc = os.path.join("data", "fold{}-raw-train-data-{}-{}-{}-{}.txt".format(fold, version, selection, classwise, proto_number))
    train_dtw_fileloc = os.path.join("data", "fold{}-dtw_features-train-data-{}-{}-{}-{}.txt".format(fold, version, selection, classwise, proto_number))
    train_combined_fileloc = os.path.join("data", "fold{}-dtw_features-plus-raw-train-data-{}-{}-{}-{}.txt".format(fold, version, selection, classwise, proto_number))

    # test set
    with open(test_label_fileloc, 'w') as test_label_file, open(test_raw_fileloc, 'w') as test_raw_file, open(
            test_dtw_fileloc, 'w') as test_dtw_file, open(test_combined_fileloc, 'w') as test_combined_file:
        writer_test_label = csv.writer(test_label_file, quoting=csv.QUOTE_NONE, delimiter=" ")
        writer_test_raw = csv.writer(test_raw_file, quoting=csv.QUOTE_NONE, delimiter=" ")
        writer_test_dtw = csv.writer(test_dtw_file, quoting=csv.QUOTE_NONE, delimiter=" ")
        writer_test_combined = csv.writer(test_combined_file, quoting=csv.QUOTE_NONE, delimiter=" ")

        for sample in range(test_number):
            local_sample = test_data[sample]
            features = get_dtwfeatures(proto_data, proto_number, local_sample)

            # set the range from 0-255 for the input_data file (the input_data file was made for images and changes it back down to -1 to 1
            features = features * 255.
            local_sample = local_sample * 255.
            class_value = test_labels[sample]

            # write files
            feature_flat = features.reshape(50 * proto_number)
            local_sample_flat = local_sample.reshape(50 * 2)
            writer_test_raw.writerow(local_sample_flat)
            writer_test_dtw.writerow(feature_flat)
            writer_test_combined.writerow(np.append(local_sample_flat, feature_flat))
            writer_test_label.writerow(["{}-{}_test.png".format(class_value, sample), class_value])
    print("{}: Test Done".format(version))

    # train set
    with open(train_label_fileloc, 'w') as train_label_file, open(train_raw_fileloc, 'w') as train_raw_file, open(
            train_dtw_fileloc, 'w') as train_dtw_file, open(train_combined_fileloc, 'w') as train_combined_file:
        writer_train_label = csv.writer(train_label_file, quoting=csv.QUOTE_NONE, delimiter=" ")
        writer_train_raw = csv.writer(train_raw_file, quoting=csv.QUOTE_NONE, delimiter=" ")
        writer_train_dtw = csv.writer(train_dtw_file, quoting=csv.QUOTE_NONE, delimiter=" ")
        writer_train_combined = csv.writer(train_combined_file, quoting=csv.QUOTE_NONE, delimiter=" ")

        for sample in range(train_number - proto_number):
            local_sample = train_data[sample]
            features = get_dtwfeatures(proto_data, proto_number, local_sample)

            # set the range from 0-255 for the input_data file (the input_data file was made for images and changes it back down to -1 to 1
            features = features * 255.
            local_sample = local_sample * 255.
            class_value = train_labels[sample]

            # write files
            feature_flat = features.reshape(50 * proto_number)
            local_sample_flat = local_sample.reshape(50 * 2)
            writer_train_raw.writerow(local_sample_flat)
            writer_train_dtw.writerow(feature_flat)
            writer_train_combined.writerow(np.append(local_sample_flat, feature_flat))
            writer_train_label.writerow(["{}-{}_train.png".format(class_value, sample), class_value])

            if sample % 1000 == 0:
                print("{}: Training < {} Done".format(version, str(sample)))
    print("{}: Training Done".format(version))


print("Done")