keras_article.py

# coding: utf-8
from __future__ import print_function
"""
created on 2017/07/06
@author: liuenda
"""

import numpy as np
import pandas as pd

import keras

from sklearn.metrics import classification_report
from gensim import corpora, models, similarities
from sklearn import datasets
from sklearn import linear_model
from sklearn import svm
from gensim.models import word2vec
from sklearn import preprocessing
import random
import pickle
import time
from keras.preprocessing import sequence
from keras.layers import Input, Embedding, LSTM, Dense, Dropout, Bidirectional
from keras.models import Model
# from copy import deepcopy

base_path = "/home/M2015eliu/cas/2017.1.1~LiuSTM/"
model_name_en = base_path + "data/model-en/W2Vmodle.bin"
model_name_jp = base_path + "data/model-jp/W2Vmodle.bin"

model_en = word2vec.Word2Vec.load(model_name_en)
model_jp = word2vec.Word2Vec.load(model_name_jp)


maxlen = 300 # Default: 0 -> infinite
epoch = 10
dim_lstm = 200
dim_1 = 800
# dim_2 = 100
# dim_3 = 50
dropout_rate = 0.0
bias_y = 0
loss_function = "mse"
mode = "reg" # reg, binary
rnn_type = "bi-lstm" # lstm, bi-lstm
bi_lstm_mode = "concat" #concat, sum
print("maxlen", maxlen, "epoch", epoch, "dim_lstm", dim_lstm)
print("dim_Dense", dim_1)
print("dropout_rate", dropout_rate, ", LSTM type:", rnn_type, bi_lstm_mode)
p_activation = ["relu", "relu", "relu"]
print("Activation function:", p_activation)
print("bias of y:", bias_y)
print("loss_function:", loss_function)
start = 0
step = 10
print("start:", start, "end:", )
print("------------------------------")
random.seed(1234)

" Padding the sequence"
def padding(sequence, maxlen=maxlen, padding_value=0.0):
    np_sequance = np.array(sequence)
    # print(np_sequance.shape)
    if np_sequance.shape[0] < maxlen:
        z = np.zeros((maxlen, 200))
        z[:np_sequance.shape[0], :np_sequance.shape[1]] = np_sequance
    else:
        z = np_sequance[:maxlen, :]
    return z

"""
Find the ranking results with respect to real pairs
Defaulty, projection1 should be JP
Whiile, projection2 should be EN->JP
"""
def find_ranking_batch(projection1, projection2, dlmodel, batch=10):
    sim_results = []
    rank_results = []
    sample_length = len(projection2)

    # Iterate each of the ariticle from projection1 (999) as proj1
    # Calculate the simialrity of proj1 with all ariticles in projection2 (999)
    # for i, proj1 in enumerate(projection1):
    for i in range(0, sample_length, batch):
        print("Find answer for doc.", i, i+batch)
        proj1 = projection1[i:i+batch]

        proj1_tile = np.repeat(proj1, sample_length, axis=0)
        proj2_tile = np.tile(projection2, (batch,1,1))
        print(proj1_tile.shape)
        print(proj2_tile.shape)

        # For each batch, we should tile each of the element
        sim = dlmodel.predict([proj1_tile, proj2_tile])[:,0]
        for j in range(0, sample_length, batch):
            rank = pd.Series(sim[j:j+sample_length]).rank(ascending = False)[i]
            sim_results.append(sim)
            rank_results.append(rank)

    # sim_results contains 999*999 similairty matrix
    return sim_results, rank_results


"""
Find the ranking results with respect to real pairs
Defaulty, projection1 should be JP
Whiile, projection2 should be EN->JP
"""
def find_ranking_quick(projection1, projection2, dlmodel):
    sim_results = []
    rank_results = []

    # ---- Prepare the model_1 ---- #

    # Input layer
    input_1 = Input(shape=(maxlen,200), dtype='float32', name='main_input_1')
    input_2 = Input(shape=(maxlen,200), dtype='float32', name='main_input_2')

    # LSTM layer
    if rnn_type == "lstm":
        lstm_out_1 = LSTM(dim_lstm, go_backwards = True)(input_1)
        lstm_out_2 = LSTM(dim_lstm, go_backwards = True)(input_2)
    elif rnn_type == "bi-lstm":
        lstm_out_1 = Bidirectional(LSTM(dim_lstm, go_backwards = True), merge_mode=bi_lstm_mode)(input_1)
        lstm_out_2 = Bidirectional(LSTM(dim_lstm, go_backwards = True), merge_mode=bi_lstm_mode)(input_2)


    # Model definition
    model_1 = Model(input=[input_1, input_2], output=[lstm_out_1, lstm_out_2])

    # Compile the model
    if mode == "reg":
        model_1.compile(optimizer='adam', loss=loss_function)
    else:
        model_1.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
    # model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])

    # Show the structure of the model
    # print model_1.summary()

    # Set the weights
    model_1.layers[2].set_weights(dlmodel.layers[2].get_weights())
    model_1.layers[3].set_weights(dlmodel.layers[3].get_weights())

    # Find the real projection
    v1, v2 = model_1.predict([projection1, projection2])

    # ---- Prepare the model_2 ----

    # Input layer
    if rnn_type == "lstm":
        co = 1
    elif rnn_type == "bi-lstm":
        co = 2
    if bi_lstm_mode == "sum" or bi_lstm_mode == "mul":
        co = 1
    input_1 = Input(shape=(dim_lstm*co,), dtype='float32', name='lstm1')
    input_2 = Input(shape=(dim_lstm*co,), dtype='float32', name='lsmt2')

    # Merge layer
    merged_vector = keras.layers.concatenate([input_1, input_2], axis=-1)

    # (Dense 1) * 3
    x1 = Dense(dim_1, activation=p_activation[0])(merged_vector)
    x1 = Dropout(dropout_rate)(x1)

    # x1 = Dense(dim_2, activation=p_activation[1])(x1)
    # x1 = Dropout(dropout_rate)(x1)
    #
    # x1 = Dense(dim_3, activation=p_activation[2])(x1)
    # x1 = Dropout(dropout_rate)(x1)

    # main_output = Dense(1, activation='sigmoid', name='main_output')(x1)
    if mode == "reg":
        main_output = Dense(1, name='main_output')(x1)
        # main_output = Dense(1, activation="sigmoid", name='main_output')(x1)
    else:
        main_output = Dense(2, activation='softmax', name='main_output')(x1)

    # Model definition
    model_2 = Model(input=[input_1, input_2], output=main_output)

    # Compile the model
    if mode == "reg":
        model_2.compile(optimizer='adam', loss=loss_function)
    else:
        model_2.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])

    # Show the structure of the model
    # print model_2.summary()

    # Find the real projection
    v3 = model_2.predict([v1, v2])

    # Set the weights
    model_2.layers[3].set_weights(dlmodel.layers[5].get_weights())
    model_2.layers[5].set_weights(dlmodel.layers[7].get_weights())
    # model_2.layers[7].set_weights(dlmodel.layers[9].get_weights())
    #
    # model_2.layers[9].set_weights(dlmodel.layers[11].get_weights())

    projection1 = v1
    projection2 = v2

    # Iterate each of the ariticle from projection1 (999) as proj1
    # Calculate the simialrity of proj1 with all ariticles in projection2 (999)
    for i, proj1 in enumerate(projection1):
        # print("Find answer for doc.", i)
        proj1_tile = np.tile(proj1, (len(projection2), 1))
        sim = model_2.predict([proj1_tile, projection2])[:,0]
        rank = pd.Series(sim).rank(ascending = False)[i]
        sim_results.append(sim)
        rank_results.append(rank)

    # sim_results contains 999*999 similairty matrix
    return sim_results, rank_results


"""
Find the ranking results with respect to real pairs
Defaulty, projection1 should be JP
Whiile, projection2 should be EN->JP
"""
def find_ranking(projection1, projection2, dlmodel):
    sim_results = []
    rank_results = []

    # Iterate each of the ariticle from projection1 (999) as proj1
    # Calculate the simialrity of proj1 with all ariticles in projection2 (999)
    for i, proj1 in enumerate(projection1):
        print("Find answer for doc.", i)
        proj1_tile = np.tile(proj1, (len(projection2), 1, 1))
        sim = dlmodel.predict([proj1_tile, projection2])[:,0]
        rank = pd.Series(sim).rank(ascending = False)[i]
        sim_results.append(sim)
        rank_results.append(rank)

    # sim_results contains 999*999 similairty matrix
    return sim_results, rank_results

"""
rank_results should be list of (999,)
"""
def find_top(rank_results, top):
    s = pd.Series(rank_results)
    n_top = (s <= top).sum()
    return n_top


def average_docment(document_embedding):
    return np.average(document_embedding, axis=0)

def sum_docment(document_embedding):
    return np.sum(document_embedding, axis=0)

# def doc2feature(corpus, tfidf, dictionary, w2v):
#     doc_features = []
#     for index, doc_bof in enumerate(corpus):
#
#         if index % 1000 == 0:
#             print(index)
#
#         doc_tfidf = tfidf[doc_bof]
#
#         doc_feature = np.zeros((200,))
#
#         for (token_id, token_tfidf) in doc_tfidf:
#             token = dictionary.get(token_id, "[unknown-id]").encode("utf-8")
#             # if token in w2v:
#             if True:
#                 token_w2v = w2v[token]
#             else:
#                 print("No word:", token)
#                 continue
#             doc_feature += token_w2v * token_tfidf
#
#         average = True
#         if average:
#             doc_feature = np.true_divide(doc_feature, len(doc_tfidf))
#         doc_features.append(doc_feature)
#
#     return doc_features


def doc2vec_en(doc):
    # r = [model_en[token] for token in doc.split()]
    r = []
    r_failed = []

    for token in doc.split():
        if token in model_en:
            r.append(model_en[token])
        else:
            r_failed.append(token)

    # if len(r_failed) != 0:
    #     print " ".join(r_failed)

    return r

def doc2vec_jp(doc):
    # r = [model_en[token] for token in doc.split()]
    r = []
    r_failed = []

    for token in doc.split():
        if token in model_jp:
            r.append(model_jp[token])
        else:
            r_failed.append(token)

    # if len(r_failed) != 0:
    #     print " ".join(r_failed)

    return r

def prepare_train(dir_en, dir_jp):

	df_en_mapping = pd.read_csv(dir_en)
	df_jp_mapping = pd.read_csv(dir_jp)

	print("Reading english Data:", len(df_en_mapping))
	print("Reading english Data:", len(df_jp_mapping))

	sample_size = len(df_en_mapping)

	assert len(df_en_mapping) == len(df_jp_mapping)

	# Convert mapping to list type and then concat to the a list
	print("Merging the English and Japanes news dataframe...")
	df_train_1 = pd.concat([df_en_mapping, df_jp_mapping], axis = 1)
	df_train_1['similarity'] = pd.Series(np.ones(sample_size,)*5)
	df_train_1['dis_similarity'] = pd.Series(np.ones(sample_size,)*1)

	# Remove null line
	print("Drop the null line...")
	# df_train_1 = df_train_1.dropna(subset=['en_article'])
	df_train_1 = df_train_1[df_train_1['en_article'] != '<NULL>']

	# Expand the training data
	en_article_wrong = df_train_1.en_article.iloc[random.sample(range(len(df_train_1)),len(df_train_1))]
	en_article_wrong.index = df_train_1.index
	print((en_article_wrong == df_train_1.en_article).value_counts())
	df_train_1['en_article_wrong'] = en_article_wrong

	# Convert dateframe to list
	train_1 = df_train_1[['en_article','jp_article','similarity']].values.tolist()
	train_2 = df_train_1[['en_article_wrong','jp_article','dis_similarity']].values.tolist()

	return train_1, train_2, df_train_1


# def prepare_train(dir_en, dir_jp):
# def prepare_train(dir_en_jp, start=None, end=None):
#
#     # df_en_mapping = pd.read_csv(dir_en)
#     # df_jp_mapping = pd.read_csv(dir_jp)
#
#     df_en_jp = pd.read_csv(dir_en_jp, names=["en_article","jp_article"], header=0)
#     df_en_mapping = df_en_jp[["en_article"]].iloc[start:end]
#     df_jp_mapping = df_en_jp[["jp_article"]].iloc[start:end]
#
#     print("Reading english Data:", len(df_en_mapping))
#     print("Reading english Data:", len(df_jp_mapping))
#
#     sample_size = len(df_en_mapping)
#
#     assert len(df_en_mapping) == len(df_jp_mapping)
#
#     # Convert mapping to list type and then concat to the a list
#     print("Merging the English and Japanes news dataframe...")
#     df_train_1 = pd.concat([df_en_mapping, df_jp_mapping], axis = 1)
#     df_train_1['similarity'] = pd.Series(np.ones(sample_size,)*5)
#     df_train_1['dis_similarity'] = pd.Series(np.ones(sample_size,)*1)
#
#     # Remove null line
#     print("Drop the null line...")
#     # df_train_1 = df_train_1.dropna(subset=['en_article'])
#     df_train_1 = df_train_1[df_train_1['en_article'] != '<NULL>']
#
#     # Expand the training data
#     en_article_wrong = df_train_1.en_article.iloc[random.sample(range(len(df_train_1)),len(df_train_1))]
#     en_article_wrong.index = df_train_1.index
#     print((en_article_wrong == df_train_1.en_article).value_counts())
#     df_train_1['en_article_wrong'] = en_article_wrong
#
#     # Convert dateframe to list
#     train_1 = df_train_1[['en_article','jp_article','similarity']].values.tolist()
#     train_2 = df_train_1[['en_article_wrong','jp_article','dis_similarity']].values.tolist()
#
#     return train_1, train_2, df_train_1

if __name__ == "__main__":


    input = 2
    # k = 10

    # --- Prepare and Loading the training data --- #

    # if input == 1:
    #     # Prepare For the training data
    #     sample_size = "_1000"
    #     dir_en = "./data/mapping/en_mapped_"+str(k) + sample_size + ".csv"
    #     dir_jp = "./data/mapping/jp_mapped_" + str(k) + sample_size + ".csv"
    #
    #     # Prepare For the test data
    #     sample_size = "_1k2k"
    #     dir_en_test = "./data/mapping/en_mapped_"+str(k) + sample_size + ".csv"
    #     dir_jp_test = "./data/mapping/jp_mapped_" + str(k) + sample_size + ".csv"
    #
    #     train_1, train_2, df_train_1 =     prepare_train(dir_en, dir_jp)
    #     test_1, test_2, df_test_1 = prepare_train(dir_en_test, dir_jp_test)

    if input == 2:
        # split_line = 5000
        # end_line = 6000
        # Prepare For the training data
        dir_en = base_path + "data/news/en_news.csv"
        dir_jp = base_path + "data/news/jp_news.csv"
        # dir_en_jp = base_path + "data_preparation/2014_cleaned_jp_en.csv"

        # pairs_correct, pairs_wrong, df_pairs = prepare_train(dir_en, dir_jp)
        # pairs_correct, pairs_wrong, df_pairs = prepare_train(dir_en_jp)
        _1, _2, df_pairs = prepare_train(dir_en, dir_jp)
        # 这里的 pairs_wrong 和 pairs_correct 都 不使用！
        # train_1 = pairs_correct[0:2000] + pairs_correct[3000:5000]
        # test_1 = pairs_correct[2000:3000]

        # train_2 = pairs_wrong[0:2000] + pairs_wrong[3000:5000]
    # test_2 = pairs_wrong[split_line:end_line]


    # Expand the training data
    # train = train_1 + train_2

    # ------- Independent data sets -------
    print("Using the new test data to evaluate.......")
    df_pairs_evaluate = df_pairs.iloc[50000:55000:5]

    df_pairs_evaluate['word2vec_en'] = df_pairs_evaluate['en_article'].apply(doc2vec_en)
    df_pairs_evaluate['word2vec_jp'] = df_pairs_evaluate['jp_article'].apply(doc2vec_jp)


    df_pairs_evaluate['padding_en'] = df_pairs_evaluate['word2vec_en'].apply(padding)
    df_pairs_evaluate['padding_jp'] = df_pairs_evaluate['word2vec_jp'].apply(padding)

    features_en_new = np.stack(df_pairs_evaluate["padding_en"].values)
    features_jp_new = np.stack(df_pairs_evaluate["padding_jp"].values)

    # --- Apply the word2vec model to the data sets --- #

    df_pairs_sample = df_pairs.iloc[0:5000]

    df_pairs_sample['word2vec_en'] = df_pairs_sample['en_article'].apply(doc2vec_en)
    df_pairs_sample['word2vec_jp'] = df_pairs_sample['jp_article'].apply(doc2vec_jp)


    # ---- Padding the vector ---- #
    df_pairs_sample['padding_en'] = df_pairs_sample['word2vec_en'].apply(padding)
    df_pairs_sample['padding_jp'] = df_pairs_sample['word2vec_jp'].apply(padding)


    # --- Prepare the training data --- #

    # Generate training data (similarity = 1)
    features_en_1 = np.stack(df_pairs_sample["padding_en"].values)
    features_jp_1 = np.stack(df_pairs_sample["padding_jp"].values)

    # Generate training data (similarity = 0)
    features_en_0 = np.array(features_en_1)
    np.random.shuffle((features_en_0))

    # check the duplicated amount
    c = np.all(features_en_1 == features_en_0, axis=(1,2))
    print("C value =", c.sum(), "position:", np.where(c== True)[0].tolist())

    # Prepare the final training and test data
    X_1 = np.concatenate((features_en_1, features_en_0), axis = 0)
    X_2 = np.concatenate((features_jp_1, features_jp_1), axis = 0)
    if mode == "reg":
        y = np.concatenate((np.ones(len(features_en_1)), np.zeros(len(features_en_0))+bias_y), axis = 0)
        # y = np.concatenate((np.ones(len(features_en_1)), np.zeros(len(features_en_0))), axis = 0)
    else:
        y = np.concatenate((np.ones(len(features_en_1)), np.zeros(len(features_en_0))), axis = 0)

    # --- Split into test data and training data --- #

    X1_train1, X1_test_1, X1_train2, X1_train3_wrong, X1_test_0 = np.split(X_1, [2000, 3000, 5000, 9000])
    X2_train1, X2_test_1, X2_train2, X2_train3_wrong, X2_test_0 = np.split(X_2, [2000, 3000, 5000, 9000])
    y_train1, y_test, y_train2, y_train3_wrong, Y_o = np.split(y, [2000, 3000, 9000, 9000])

    X1_train = np.concatenate((X1_train1, X1_train2, X1_train3_wrong), axis = 0)
    X2_train = np.concatenate((X2_train1, X2_train2, X2_train3_wrong), axis = 0)
    y_train = np.concatenate((y_train1, y_train2, y_train3_wrong), axis = 0)
    # X_train_correct = np.concatenate((X_train1, X_train2), axis = 0)
    # y_train_correct = np.concatenate((y_train1, y_train2), axis = 0)

    # --- Generate balanced test data --- #
    X1_test = np.concatenate((X1_test_1, X1_test_0), axis=0)
    X2_test = np.concatenate((X2_test_1, X2_test_0), axis=0)
    if mode == "reg":
        y_test = np.concatenate((np.ones(len(X1_test_1)), np.zeros(len(X1_test_0))+bias_y), axis = 0)
        # y_test = np.concatenate((np.ones(len(X1_test_1)), np.zeros(len(X1_test_0))), axis = 0)
    else:
        y_test = np.concatenate((np.ones(len(X1_test_1)), np.zeros(len(X1_test_0))), axis = 0)


    # --- Generate balanced test data 2 --- #

    # Generate training data (similarity = 0)
    features_en_0_new = np.array(features_en_new)
    np.random.shuffle((features_en_0_new))

    # check the duplicated amount
    c = np.all(features_en_new == features_en_0_new, axis=(1,2))
    print("C value =", c.sum(), "position:", np.where(c== True)[0].tolist())

    # Prepare the final training and test data
    X1_test_new = np.concatenate((features_en_new, features_en_0_new), axis = 0)
    X2_test_new = np.concatenate((features_jp_new, features_jp_new), axis = 0)

    if mode == "reg":
        y_test_new = np.concatenate((np.ones(len(features_en_new)), np.zeros(len(features_en_0_new))+bias_y), axis = 0)
        # y_test = np.concatenate((np.ones(len(X1_test_1)), np.zeros(len(X1_test_0))), axis = 0)
    else:
        y_test_new = np.concatenate((np.ones(len(features_en_new)), np.zeros(len(features_en_0_new))), axis = 0)


    # ---- Parallel Model ---- #

    # Input layer
    input_1 = Input(shape=(maxlen,200), dtype='float32', name='main_input_1')
    input_2 = Input(shape=(maxlen,200), dtype='float32', name='main_input_2')

    # LSTM layer
    # lstm_out_1 = LSTM(50)(input_1)
    # lstm_out_2 = LSTM(50)(input_2)
    if rnn_type == "lstm":
        lstm_out_1 = LSTM(dim_lstm, go_backwards = True)(input_1)
        lstm_out_2 = LSTM(dim_lstm, go_backwards = True)(input_2)
    elif rnn_type == "bi-lstm":
        lstm_out_1 = Bidirectional(LSTM(dim_lstm, go_backwards = True), merge_mode=bi_lstm_mode)(input_1)
        lstm_out_2 = Bidirectional(LSTM(dim_lstm, go_backwards = True), merge_mode=bi_lstm_mode)(input_2)

    # Merge layer
    merged_vector = keras.layers.concatenate([lstm_out_1, lstm_out_2], axis=-1)

    # (Dense 1) * 3
    x1 = Dense(dim_1, activation=p_activation[0])(merged_vector)
    x1 = Dropout(dropout_rate)(x1)

    # x1 = Dense(dim_2, activation=p_activation[1])(x1)
    # x1 = Dropout(dropout_rate)(x1)
    #
    # x1 = Dense(dim_3, activation=p_activation[2])(x1)
    # x1 = Dropout(dropout_rate)(x1)

    if mode == "reg":
        main_output = Dense(1, name='main_output')(x1)
        # main_output = Dense(1, activation="sigmoid", name='main_output')(x1)
    else:
        main_output = Dense(2, activation='softmax', name='main_output')(x1)

    # Model definition
    model_lstm2 = Model(input=[input_1, input_2], output=main_output)

    # Compile the model
    # model_lstm2.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
    model_lstm2.compile(optimizer='adam', loss=loss_function)
    # model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])

    # Show the structure of the model
    print(model_lstm2.summary())

    saved_model = []
    tops = {}
    tops_test = {}
    # Initialize the top saver
    for i in [1,5,10]:
        tops[i] = []
        tops_test[i] = []

    # Fit the training model
    for i in range(epoch):
        hist = model_lstm2.fit([X1_train, X2_train], [y_train],
                           validation_data=([X1_test, X2_test], y_test),
                           epochs=i+1,
                           batch_size=32,
                           initial_epoch=i)

        # Evaluation on test data 1
        sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,
                                                                 X2_test_1,
                                                                 model_lstm2)

        # Evaluation on test data 2
        sim_results_test2, rank_results_test2 = find_ranking_quick(features_en_new,
                                                             features_jp_new,
                                                             model_lstm2)

        # Save the mdoel
        saved_model.append(model_lstm2.get_weights())

        # Evaluation

        for i in [1,5,10]:
            top = (pd.Series(rank_results_test) <= i).sum()
            tops[i].append(top)
            print("TOP", i, top)

            top_test = (pd.Series(rank_results_test2) <= i).sum()
            tops_test[i].append(top_test)
            print("[T] TOP", i, top_test)

    # # Save the history and the model
    # code = "b"
    # path_model_lstm2 = "model_lstm2_" + code
    # model_lstm2.save(path_model_lstm2)
    # path_hist = "hist_lstm2_" + code
    # f = open(path_hist, "wb")
    # pickle.dump(hist.history, f)
    # f.close()

    # ---- 看样子主要的时间都花在了lstm上吧？ --- #
    # 想办法保存lstm以后的映射 --- #

    # --- find ranking --- #
    # sim_results_test_slow, rank_results_test_slow = find_ranking(X1_test_1, X2_test_1, model_lstm2)
    # sim_results_test, rank_results_test = find_ranking_quick(X1_test_1, X2_test_1, model_lstm2)
    # print("TOP1", (pd.Series(rank_results_test) <= 1).sum())
    # print("TOP5", (pd.Series(rank_results_test) <= 5).sum())
    # print("TOP10", (pd.Series(rank_results_test) <= 10).sum())

    # # --- Prepare for a new independent evaluation balanced data --- #
    #
    # df_pairs_evaluate = df_pairs.iloc[55000:60000]
    #
    # df_pairs_evaluate['word2vec_en'] = df_pairs_evaluate['en_article'].apply(doc2vec_en)
    # df_pairs_evaluate['word2vec_jp'] = df_pairs_evaluate['jp_article'].apply(doc2vec_jp)
    #
    # features_en_eva = doc2feature(corpus_en[60000:61000], tfidf_en, dictionary_en, model_en)
    # features_jp_eva = doc2feature(corpus_jp[60000:61000], tfidf_jp, dictionary_jp, model_jp)
    #
    # features_merge_eva = np.concatenate((features_en_eva,features_jp_eva), axis = 1)
    #
    # features_en_wrong_eva =  features_en[:1000]
    # # features_en_wrong_eva = np.array(features_en_eva)
    # # np.random.shuffle((features_en_wrong_eva))
    # # c = np.all(features_en_wrong_eva == features_en_eva, axis=1)
    # # print "C value =", c.sum() # check the duplicated amount
    #
    # features_merge_wrong = np.concatenate((features_en_wrong_eva,features_jp_eva), axis = 1)
    #
    # X_eva = np.concatenate((features_merge_eva, features_merge_wrong), axis = 0)
    # y_eva = np.concatenate((np.ones(len(features_merge_eva)), np.zeros(len(features_en_wrong_eva))), axis = 0)
    #
    # y_eva_predict = clf.predict(X_eva)
    #
    # print("classification report of TRAINING data:")
    # print(classification_report(y_eva, y_eva_predict))
    #
    #
    # # --- Evaluation for SVM --- #
    #
    # y_test_proba = clf.predict_proba(X_test)
    # y_train_proba = clf.predict_proba(X_train)
    #
    # # sim_results_train, rank_results_train = find_ranking(projection1_train, projection2_train)
    # sim_results_test, rank_results_test = find_ranking(X_test[:,:200] ,X_test[:,200:], clf)
    #
    #
    # print(pd.Series(rank_results_test).describe())