Kuzushiji-MNIST dataset (KMNIST)

import CSV
import NPZ
import PyPlot
import Gadfly
import Images

function getCategoricalCount(InputArray)
    Count = Dict{Any,Int64}()
    for (Index, Category) in enumerate(InputArray)
        if haskey(Count, Category)
            Count[Category] +=1
        else
            Count[Category] = 1
        end
    end
    return sort(Count)
end

function getCategoricalIndices(InputArray)
    CategoricalIndices = Dict{Any,Array{Int64,1}}()
    for (Index, Category) in enumerate(InputArray)
        if haskey(CategoricalIndices, Category)
            push!(CategoricalIndices[Category], Index)
        else
            CategoricalIndices[Category] = [Index]
        end
    end
    return CategoricalIndices
end

function CollectAllImagesPerCategory(InputArray,CategoricalIndices,NumberOfExamples)
    if NumberOfExamples > size(CategoricalIndices)[1]
        NumberOfExamples = size(CategoricalIndices)[1]
    end
    ItemsXY = convert(Int64,ceil(sqrt(NumberOfExamples)))
    ImageSizeY = size(InputArray)[2]
    ImageSizeX = size(InputArray)[3]
    i = 1
    FirstRow = true
    FirstItemInRow = true
    OutputArrayRow = []
    OutputArray = []  
    for j in collect(1:ItemsXY)
        for k in collect(1:ItemsXY)
            if (FirstItemInRow == true)
                OutputArrayRow = InputArray[CategoricalIndices[i],:,:]
                OutputArrayRow = hcat(OutputArrayRow, zeros(ImageSizeY,ImageSizeX))
                FirstItemInRow = false
                i+=1
            elseif (i <= NumberOfExamples)
                OutputArrayRow = hcat(OutputArrayRow,InputArray[CategoricalIndices[i],:,:])
                OutputArrayRow = hcat(OutputArrayRow, zeros(ImageSizeY,ImageSizeX))
                i+=1
            else
                OutputArrayRow = hcat(OutputArrayRow, zeros(ImageSizeY,ImageSizeX * 2))
                i+=1
            end
        end
        FirstItemInRow = true
        if FirstRow == false
            OutputArray = vcat(OutputArray, OutputArrayRow)
            OutputArray = vcat(OutputArray, zeros(ImageSizeY,ImageSizeX*2*ItemsXY))
        else
            OutputArray = zeros(ImageSizeY,ImageSizeX*2*ItemsXY)
            OutputArray = vcat(OutputArray,OutputArrayRow)
            OutputArray = vcat(OutputArray, zeros(ImageSizeY,ImageSizeX*2*ItemsXY))
            FirstRow = false
        end
    end
    return hcat(zeros(size(OutputArray)[1],ImageSizeX),OutputArray)
end

function PlotExampleImagesPerCategory(InputArray, CategoricalIndices, NumberOfExamples,DatasetName)
    for i in collect(keys(CategoricalIndices))
        Examples = CollectAllImagesPerCategory(InputArray,CategoricalIndices[i],NumberOfExamples)
        PyPlot.figure(figsize=(12,12))
        PyPlot.imshow(Examples, cmap="gray")
        PyPlot.title("Examples of class "*string(i)*" of "*DatasetName)
        PyPlot.xticks([])
        PyPlot.yticks([])
        PyPlot.tight_layout()
        PyPlot.savefig("./graphics/"*DatasetName*"_examples_class_"*string(i)*".png")
        PyPlot.close()
    end
end

function PlotHistogram(Categories, Title, XLabel="Class", YLabel="Frequency")
    Plot = Gadfly.plot(x=[string(i) for i in collect(keys(Categories))],
        y=collect(values(Categories)),
        Gadfly.Geom.bar,
        color=[string(i) for i in collect(keys(Categories))],
        Gadfly.Guide.xlabel(XLabel),
        Gadfly.Guide.ylabel(YLabel),
        Gadfly.Guide.title(Title * " Histogram"),
        Gadfly.Guide.colorkey(title=XLabel));
    ImageParams = Gadfly.SVG("./graphics/"*Title*"_histogram.svg", 14Gadfly.cm, 10Gadfly.cm);
    Gadfly.draw(ImageParams, Plot);    
end

ClassLabels_Kuzushiji_MNIST = CSV.File("./Kuzushiji-MNIST/data/kmnist_classmap.csv") |> CSV.DataFrame

Kuzushiji_MNIST_X_train = NPZ.npzread("./Kuzushiji-MNIST/data/kmnist-train-imgs.npz")["arr_0"];
Kuzushiji_MNIST_X_test = NPZ.npzread("./Kuzushiji-MNIST/data/kmnist-test-imgs.npz")["arr_0"];
Kuzushiji_MNIST_y_train = NPZ.npzread("./Kuzushiji-MNIST/data/kmnist-train-labels.npz")["arr_0"];
Kuzushiji_MNIST_y_test = NPZ.npzread("./Kuzushiji-MNIST/data/kmnist-test-labels.npz")["arr_0"];

PyPlot.figure(figsize=(1,1));
PyPlot.imshow(Kuzushiji_MNIST_X_train[1,:,:], cmap="gray");
PyPlot.show();

Kuzushiji_MNIST_CategoricalCount = getCategoricalCount(Kuzushiji_MNIST_y_train);
Kuzushiji_MNIST_CategoricalIndices = getCategoricalIndices(Kuzushiji_MNIST_y_train);
PlotExampleImagesPerCategory(Kuzushiji_MNIST_X_train, Kuzushiji_MNIST_CategoricalIndices,144, "Kuzushiji-MNIST")
PlotHistogram(Kuzushiji_MNIST_CategoricalCount, "Kuzushiji-MNIST Histogram")

ClassLabels_Kuzushiji_49 = CSV.File("./Kuzushiji-49/data/k49_classmap.csv") |> CSV.DataFrame

Kuzushiji_49_X_train = NPZ.npzread("./Kuzushiji-49/data/k49-train-imgs.npz")["arr_0"];
Kuzushiji_49_X_test = NPZ.npzread("./Kuzushiji-49/data/k49-test-imgs.npz")["arr_0"];
Kuzushiji_49_y_train = NPZ.npzread("./Kuzushiji-49/data/k49-train-labels.npz")["arr_0"];
Kuzushiji_49_y_test = NPZ.npzread("./Kuzushiji-49/data/k49-test-labels.npz")["arr_0"];

Kuzushiji_49_CategoricalCount = getCategoricalCount(Kuzushiji_49_y_train);
Kuzushiji_49_CategoricalIndices = getCategoricalIndices(Kuzushiji_49_y_train);
PlotExampleImagesPerCategory(Kuzushiji_49_X_train, Kuzushiji_49_CategoricalIndices,144, "Kuzushiji-49")
PlotHistogram(Kuzushiji_49_CategoricalCount, "Kuzushiji-49 Histogram")

function ReadFilePaths(PathToDataFolder)
    OutputDict = Dict{String,Array{String,1}}()
    SymbolFolders = readdir(PathToDataFolder);
    for SymbolFolder in SymbolFolders
        OutputDict[SymbolFolder] = readdir(PathToDataFolder * SymbolFolder)
    end
    return OutputDict
end

function getClassCountKanji(FilePaths, XLabel="Class", YLabel="Frequency")
    FrequencyCountDict = Dict{String,Int64}();
    for Symbol in collect(keys(Kuzushiji_Kanji_SymbolsWithFilePaths))
        FrequencyCountDict[Symbol] = size(Kuzushiji_Kanji_SymbolsWithFilePaths[Symbol])[1]
    end
    return FrequencyCountDict
end

Kuzushiji_Kanji_SymbolsWithFilePaths = ReadFilePaths(PathToDataFolder);
collect(values(getClassCountKanji(Kuzushiji_Kanji_SymbolsWithFilePaths)))

function KanjiExamplePlots(InputDict, NumberofXSubPlots)
    DictLength = length(InputDict);
    PlotXDim = NumberofXSubPlots*2 + 1;
    PlotYDim = 2 * div(DictLength,NumberofXSubPlots);
    PyPlot.figure(figsize=(PlotXDim, PlotYDim));
    PositionCounter = 1;
    for Symbol in collect(keys(InputDict))
        PyPlot.subplot(convert(Int64,ceil(div(DictLength,NumberofXSubPlots))+1), NumberofXSubPlots, PositionCounter)
        PyPlot.imshow(convert(Array{Float64,2},Images.load(PathToDataFolder*Symbol*"/"*InputDict[Symbol][1])),
            cmap="gray")
        PyPlot.title("Example of "*Symbol)
        PyPlot.xticks([])
        PyPlot.yticks([])
        PositionCounter += 1
    end
    PyPlot.tight_layout()
    PyPlot.savefig("./graphics/KanjiExamples.png")
    PyPlot.close()
end


KanjiExamplePlots(Kuzushiji_Kanji_SymbolsWithFilePaths,15)

import ScikitLearn
import Flux

# dataset in 1D for ScikitLearn
Kuzushiji_MNIST_X_train_flat = reshape(Kuzushiji_MNIST_X_train, (size(Kuzushiji_MNIST_X_train)[1],size(Kuzushiji_MNIST_X_train)[2]*size(Kuzushiji_MNIST_X_train)[3]));
Kuzushiji_MNIST_X_test_flat = reshape(Kuzushiji_MNIST_X_test, (size(Kuzushiji_MNIST_X_test)[1],size(Kuzushiji_MNIST_X_test)[2]*size(Kuzushiji_MNIST_X_test)[3]));

ScikitLearn.@sk_import neighbors: KNeighborsClassifier;
GridSCV = ScikitLearn.GridSearch.GridSearchCV;


KNNClassifier = KNeighborsClassifier(n_jobs=-1)
GridParameters = Dict{String, Array}(
    "n_neighbors" => [3 4 5 6 7 8 9],
    "weights" => ["uniform", "distance"],
    "algorithm" => ["ball_tree", "kd_tree", "brute"],
    "leaf_size" => [15 30 45 60],
    "p" => [1 2 3 4 5] #using Manhattan distance (1), Euclidean distance (2), arbitray Minkowski distances
    );
GridObject = GridSCV(
    KNNClassifier,
    GridParameters,
    cv=5,
    verbose=2
    # n_jobs=-1 is not implemented yet :(
    );
GridFit = ScikitLearn.fit!(
    GridObject,
    Kuzushiji_MNIST_X_train_flat,
    Kuzushiji_MNIST_y_train
    )

KNNClassifier = KNeighborsClassifier(n_jobs=-1)
GridParameters = Dict{String, Array}(
    "n_neighbors" => [3 4],#[3 4 5 6 7 8 9],
    #"weights" => #["uniform", "distance"],
    #"algorithm" => #["ball_tree", "kd_tree", "brute"],
    #"leaf_size" => #[15 30 45 60],
    "p" => [1 2]#[1 2 3 4 5] #using Manhattan distance (1), Euclidean distance (2), arbitray Minkowski distances
    );
GridObject = GridSCV(
    KNNClassifier,
    GridParameters,
    cv=5,
    verbose=2,
    # n_jobs=-1 is not implemented yet :(
    );
GridFit = ScikitLearn.fit!(
    GridObject,
    Kuzushiji_MNIST_X_train_flat,
    Kuzushiji_MNIST_y_train
    )

GridFit.best_params_
Dict{Symbol,Any} with 2 entries:
  :n_neighbors => 3
  :p           => 2

Kuzushiji_MNIST_y_test_pred = ScikitLearn.predict(GridFit,Kuzushiji_MNIST_X_test_flat);
Kuzushiji_MNIST_accuracy = sum(Kuzushiji_MNIST_y_test_pred .== Kuzushiji_MNIST_y_test)/length(Kuzushiji_MNIST_y_test_pred)