# Variables

# Embedding layers, contains the word embeddings

# We define two embedding layers

# in_embeddings is used to lookup embeddings corresponding to target words (inputs)

in_embeddings = tf.Variable(

    tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)

)

# out_embeddings is used to lookup embeddings corresponding to contect words (labels)

out_embeddings = tf.Variable(

    tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0)

 改进（或者优化）后的Skip-gram实现的代码   

# Variables

# Embedding layer, contains the word embeddings

embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))

# Softmax Weights and Biases

softmax_weights = tf.Variable(

    tf.truncated_normal([vocabulary_size, embedding_size],

                        stddev=0.5 / math.sqrt(embedding_size))

)

softmax_biases = tf.Variable(tf.random_uniform([vocabulary_size],0.0,0.01)

# 1. Compute negative sampels for a given batch of data

# Returns a [num_sampled] size Tensor

negative_samples, _, _ = tf.nn.log_uniform_candidate_sampler(train_labels, num_true=1, num_sampled=num_sampled,

                                                           unique=True, range_max=vocabulary_size)

# 2. Look up embeddings for inputs, outputs and negative samples.

in_embed = tf.nn.embedding_lookup(in_embeddings, train_dataset)

out_embed = tf.nn.embedding_lookup(out_embeddings, tf.reshape(train_labels,[-1]))

negative_embed = tf.nn.embedding_lookup(out_embeddings, negative_samples) 

# 3. Manually defining negative sample loss

# As Tensorflow have a limited amount of flexibility in the built-in sampled_softmax_loss function,

# we have to manually define the loss fuction.

# 3.1. Computing the loss for the positive sample

# Exactly we compute log(sigma(v_o * v_i^T)) with this equation

loss = tf.reduce_mean(

  tf.log(

      tf.nn.sigmoid(

          tf.reduce_sum(

              tf.diag([1.0 for _ in range(batch_size)])*

              tf.matmul(out_embed,tf.transpose(in_embed)),

          axis=0)

      )

  )

)

# 3.2. Computing loss for the negative samples

# We compute sum(log(sigma(-v_no * v_i^T))) with the following

# Note: The exact way this part is computed in TensorFlow library appears to be

# by taking only the weights corresponding to true samples and negative samples

# and then computing the softmax_cross_entropy_with_logits for that subset of weights.

# More infor at: https://github.com/tensorflow/tensorflow/blob/r1.8/tensorflow/python/ops/nn_impl.py

# Though the approach is different, the idea remains the same

loss += tf.reduce_mean(

  tf.reduce_sum(

      tf.log(tf.nn.sigmoid(-tf.matmul(negative_embed,tf.transpose(in_embed)))),

      axis=0

  )

)

# The above is the log likelihood.

# We would like to transform this to the negative log likelihood

# to convert this to a loss. This provides us with

# L = - (log(sigma(v_o * v_i^T))+sum(log(sigma(-v_no * v_i^T))))

loss *= -1.0

Define Label colors for %d 20

Running K-Means for skip-gram

Running K-Means for CBOW

K-Means ran successfully

pylab.figure(figsize=(15,5))  # in inches

# Define the x axis

x = np.arange(len(skip_gram_loss))*2000

# Plotting standard CBOW loss, CBOW loss with unigram sampling and

# CBOW loss with unigram sampling + subsampling here in one plot

pylab.plot(x, cbow_loss, label="CBOW",linestyle='--',linewidth=2)

pylab.plot(x, cbow_loss_unigram, label="CBOW (Unigram)",linestyle='-.',linewidth=2,marker='^',markersize=5)

pylab.plot(x, cbow_loss_unigram_subsampled, label="CBOW (Unigram+Subsampling)",linewidth=2)

# Some text around the plots

pylab.title('Original CBOW vs Various Improvements Loss Decrease Over-Time',fontsize=24)

pylab.xlabel('Iterations',fontsize=22)

pylab.ylabel('Loss',fontsize=22)

pylab.legend(loc=1,fontsize=22)

# Use for saving the figure if needed

pylab.savefig('loss_cbow_vs_all_improvements.png')

pylab.show()

num_steps = 100001

glove_loss = []

average_loss = 0

with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as session:

    tf.global_variables_initializer().run()

    print('Initialized')

    for step in range(num_steps):

        # generate a single batch (data,labels,co-occurance weights)

        batch_data, batch_labels, batch_weights = generate_batch(

            batch_size, skip_window) 

        # Computing the weights required by the loss function

        batch_weights = [] # weighting used in the loss function

        batch_xij = [] # weighted frequency of finding i near j

        # Compute the weights for each datapoint in the batch

        for inp,lbl in zip(batch_data,batch_labels.reshape(-1)):

            point_weight = (cooc_mat[inp,lbl]/100.0)**0.75 if cooc_mat[inp,lbl]<100.0 else 1.0

            batch_weights.append(point_weight)

            batch_xij.append(cooc_mat[inp,lbl])

        batch_weights = np.clip(batch_weights,-100,1)

        batch_xij = np.asarray(batch_xij)

        # Populate the feed_dict and run the optimizer (minimize loss)

        # and compute the loss. Specifically we provide

        # train_dataset/train_labels: training inputs and training labels

        # weights_x: measures the importance of a data point with respect to how much those two words co-occur

        # x_ij: co-occurence matrix value for the row and column denoted by the words in a datapoint

        feed_dict = {train_dataset : batch_data.reshape(-1), train_labels : batch_labels.reshape(-1),

                    weights_x:batch_weights,x_ij:batch_xij}

        _, l = session.run([optimizer, loss], feed_dict=feed_dict)

        # Update the average loss variable

        average_loss += l

        if step % 2000 == 0:

          if step > 0:

            average_loss = average_loss / 2000

          # The average loss is an estimate of the loss over the last 2000 batches.

          print('Average loss at step %d: %f' % (step, average_loss))

          glove_loss.append(average_loss)

          average_loss = 0

        # Here we compute the top_k closest words for a given validation word

        # in terms of the cosine distance

        # We do this for all the words in the validation set

        # Note: This is an expensive step

        if step % 10000 == 0:

          sim = similarity.eval()

          for i in range(valid_size):

            valid_word = reverse_dictionary[valid_examples[i]]

            top_k = 8 # number of nearest neighbors

            nearest = (-sim[i, :]).argsort()[1:top_k+1]

            log = 'Nearest to %s:' % valid_word

            for k in range(top_k):

              close_word = reverse_dictionary[nearest[k]]

              log = '%s %s,' % (log, close_word)

            print(log)

    final_embeddings = normalized_embeddings.eval()

Initialized

Average loss at step 0: 9.578778

Nearest to it: karol, burgh, destabilise, armchair, crook, roguery, one-sixth, swains,

Nearest to that: wmap, partake, ahmadi, armstrong, memberships, forza, director-general, condo,

Nearest to has: mentality, vastly, approaches, bulwark, enzymes, originally, privatize, reunify,

Nearest to but: inhabited, potrero, trust, memory, curran, philips, p.m.s, pagoda,

Nearest to city: seals, counter-revolution, tubular, kayaking, central, 1568, override, buckland,

Nearest to this: dispersion, intermarriage, dialysis, moguls, aldermen, alcoholic, codes, farallon,

Nearest to UNK: 40.3, tatsam, jupiter, verify, unequal, berliners, march, 1559,

Nearest to by: functionalists, synthesised, palladius, chiapas, synaptic, sumner, raining, valued,

Nearest to or: amherst, 'mother, epiglottis, wen, stanislaus, trafford, cuticle, reminded,

Nearest to been: 640,961., depression-era, uniquely, mami, 375,000, stickiness, medium-sized, amor,

Nearest to with: anti-statist, pitigliano, branches, reparations, acquittal, frowned, pishpek, left-leaning,

Nearest to be: i-20, kevin, greased, rightly, conductors, hypercholesterolemia, pedro, douaumont,

Nearest to as: gabon, horda, mead, protruding, soundtrack, algeria, 48, macon,

Nearest to at: kambula, tisa, spelled, 130,000, 2008, organisers, |jul_rec_lo_°f, arrows,

Nearest to ,: is, of, its, malton, martinů, retiree, reliant, uri,

Nearest to its: of, ,, galleon, gitlow, rugby-playing, varanasi, fono, clusters,

Average loss at step 2000: 0.739107

Average loss at step 4000: 0.091107

Average loss at step 6000: 0.068614

Average loss at step 8000: 0.076040

Average loss at step 10000: 0.058149

Nearest to it: was, is, that, not, a, in, to, .,

Nearest to that: is, was, the, a, ., ,, to, in,

Nearest to has: is, it, that, a, been, was, to, mentality,

Nearest to but: with, said, trust, mating, not, squamous, war—the, r101,

Nearest to city: of, 's, counter-revolution, the, professed, ., equilibrium, seals,

Nearest to this: is, ., for, in, was, the, a, that,

Nearest to UNK: and, ,, (, in, the, ., ), a,

Nearest to by: the, and, ,, ., in, was, of, a,

Nearest to or: UNK, ,, and, a, cuticle, donnchad, ``, 'mother,

Nearest to been: have, had, to, has, be, was, that, it,

Nearest to with: ,, and, a, the, in, of, for, .,

Nearest to by: the, was, ,, in, ., and, a, of,

Nearest to or: (, UNK, ), ``, a, ,, and, with,

Nearest to been: have, has, had, also, be, that, was, to,

Nearest to with: and, ,, a, the, of, in, for, .,

Nearest to be: to, have, can, not, that, from, is, would,

Nearest to as: a, an, ,, such, for, and, is, the,

Nearest to at: of, the, ., in, 's, ,, and, by,

Nearest to ,: and, in, the, ., a, with, of, UNK,

Nearest to its: for, and, their, with, his, ,, the, of,

Average loss at step 92000: 0.019305

Average loss at step 94000: 0.019555

Average loss at step 96000: 0.019266

Average loss at step 98000: 0.018803

Average loss at step 100000: 0.018488

Nearest to it: is, was, also, that, not, has, this, a,

Nearest to that: was, is, to, it, the, a, ., ,,

Nearest to has: it, been, was, had, also, is, that, a,

Nearest to but: which, not, ,, it, with, was, and, a,

Nearest to city: of, 's, the, ., in, is, new, world,

Nearest to this: is, ., was, it, in, for, the, at,

Nearest to UNK: (, and, ), ,, or, a, the, .,

Nearest to by: the, ., was, ,, and, in, of, a,

Nearest to or: UNK, (, ``, a, ), ,, and, with,

Nearest to been: have, has, had, also, be, was, that, to,

Nearest to with: and, ,, a, the, of, in, for, .,

Nearest to be: to, have, can, not, would, from, that, a,

Nearest to as: a, such, an, ,, for, is, and, to,

Nearest to at: of, ., the, in, 's, by, ,, and,

Nearest to ,: and, in, the, ., a, with, UNK, of,

Nearest to its: for, their, and, with, his, ,, to, the,

num_steps = 100001

cbow_loss = []

config=tf.ConfigProto(allow_soft_placement=True)

# This is an important setting and with limited GPU memory,

# not using this option might lead to the following error.

# InternalError (see above for traceback): Blas GEMM launch failed : ...

config.gpu_options.allow_growth = True

with tf.Session(config=config) as session:

    # Initialize the variables in the graph

    tf.global_variables_initializer().run()

    print('Initialized')

    average_loss = 0

    # Train the Word2vec model for num_step iterations

    for step in range(num_steps):

        # Generate a single batch of data

        batch_data, batch_labels = generate_batch(data, batch_size, window_size)

        # Populate the feed_dict and run the optimizer (minimize loss)

        # and compute the loss

        feed_dict = {train_dataset : batch_data, train_labels : batch_labels}

        _, l = session.run([optimizer, loss], feed_dict=feed_dict)

        # Update the average loss variable

        average_loss += l

        if (step+1) % 2000 == 0:

            if step > 0:

                average_loss = average_loss / 2000

                # The average loss is an estimate of the loss over the last 2000 batches.

            print('Average loss at step %d: %f' % (step+1, average_loss))

            cbow_loss.append(average_loss)

            average_loss = 0

        # Evaluating validation set word similarities

        if (step+1) % 10000 == 0:

            sim = similarity.eval()

            # Here we compute the top_k closest words for a given validation word

            # in terms of the cosine distance

            # We do this for all the words in the validation set

            # Note: This is an expensive step

            for i in range(valid_size):

                valid_word = reverse_dictionary[valid_examples[i]]

                top_k = 8 # number of nearest neighbors

                nearest = (-sim[i, :]).argsort()[1:top_k+1]

                log = 'Nearest to %s:' % valid_word

                for k in range(top_k):

                    close_word = reverse_dictionary[nearest[k]]

                    log = '%s %s,' % (log, close_word)

                print(log)

    # Computing test documents embeddings by averaging word embeddings

    # We take batch_size*num_test_steps words from each document

    # to compute document embeddings

    num_test_steps = 100

    # Store document embeddings

    # {document_id:embedding} format

    document_embeddings = {}

    print('Testing Phase (Compute document embeddings)')

    # For each test document compute document embeddings

    for k,v in test_data.items():

        print('\tCalculating mean embedding for document ',k,' with ', num_test_steps, ' steps.')

        test_data_index = 0

        topic_mean_batch_embeddings = np.empty((num_test_steps,embedding_size),dtype=np.float32)

        # keep averaging mean word embeddings obtained for each step

        for test_step in range(num_test_steps):

            test_batch_labels = generate_test_batch(test_data[k],batch_size)

            batch_mean = session.run(mean_batch_embedding,feed_dict={test_labels:test_batch_labels})

            topic_mean_batch_embeddings[test_step,:] = batch_mean

        document_embeddings[k] = np.mean(topic_mean_batch_embeddings,axis=0)

输出（给出部分内容）     

# Create and fit K-means

kmeans = KMeans(n_clusters=5, random_state=43643, max_iter=10000, n_init=100, algorithm='elkan')

kmeans.fit(np.array(list(document_embeddings.values())))

# Compute items fallen within each cluster

document_classes = {}

for inp, lbl in zip(list(document_embeddings.keys()), kmeans.labels_):

    if lbl not in document_classes:

        document_classes[lbl] = [inp]

    else:

        document_classes[lbl].append(inp)

for k,v in document_classes.items():

    print('\nDocuments in Cluster ',k)

    print('\t',v)

Documents in Cluster  0

     ['entertainment-216', 'business-240', 'business-44', 'tech-178', 'business-165', 'tech-238', 'business-171', 'business-144', 'business-107']

Documents in Cluster  1

     ['tech-34', 'tech-145', 'business-135', 'sport-206', 'tech-216', 'politics-184', 'politics-247', 'politics-171', 'politics-8', 'politics-78', 'entertainment-163', 'politics-16', 'business-141', 'business-215', 'tech-79', 'tech-157', 'sport-231', 'tech-42', 'politics-197', 'politics-98', 'tech-212']

Documents in Cluster  2

     ['sport-166', 'entertainment-119', 'business-161', 'sport-129', 'sport-45', 'entertainment-98', 'entertainment-196', 'politics-236', 'sport-26', 'entertainment-1', 'entertainment-74', 'entertainment-244', 'entertainment-154']

Documents in Cluster  3

     ['sport-184']

Documents in Cluster  4

     ['sport-87', 'sport-32', 'sport-20']