Learning_Pytorch/RNN_word_embeddings.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"RNN_word_embeddings.ipynb","version":"0.3.2","provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"id":"NhAFWp6US47T","colab_type":"code","outputId":"d8989743-eec1-4c0b-f0ff-fbd2d4673298","executionInfo":{"status":"error","timestamp":1554378447966,"user_tz":-180,"elapsed":905,"user":{"displayName":"Omer Sezer","photoUrl":"","userId":"08295833296445258983"}},"colab":{"base_uri":"https://localhost:8080/","height":73}},"source":["!apt-get install -y -qq software-properties-common python-software-properties module-init-tools\n","!add-apt-repository -y ppa:alessandro-strada/ppa 2>&1 > /dev/null\n","!apt-get update -qq 2>&1 > /dev/null\n","!apt-get -y install -qq google-drive-ocamlfuse fuse\n","from google.colab import auth\n","auth.authenticate_user()\n","from oauth2client.client import GoogleCredentials\n","creds = GoogleCredentials.get_application_default()\n","import getpass\n","!google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret} < /dev/null 2>&1 | grep URL\n","vcode = getpass.getpass()\n","!echo {vcode} | google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret}"],"execution_count":0,"outputs":[{"output_type":"stream","text":["E: Package 'python-software-properties' has no installation candidate\n","^C\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"IKF4ZrtFDCWz","colab_type":"code","colab":{}},"source":["from google.colab import auth\n","auth.authenticate_user()\n","from oauth2client.client import GoogleCredentials\n","creds = GoogleCredentials.get_application_default()\n","import getpass\n","!google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret} < /dev/null 2>&1 | grep URL\n","vcode = getpass.getpass()\n","!echo {vcode} | google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret}"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"mB90VTcQTX1X","colab_type":"code","colab":{}},"source":["!mkdir -p drive\n","!google-drive-ocamlfuse drive\n","import sys\n","sys.path.insert(0,'drive/Fast-Pytorch/Learning_Pytorch')\n","!ls drive"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"smIdEcKsyaqZ","colab_type":"code","colab":{}},"source":["import torch\n","import os\n","import torch.nn as nn\n","import numpy as np\n","from torch.nn.utils import clip_grad_norm"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"MDTo_z4ey2Mk","colab_type":"code","colab":{}},"source":["class Dictionary(object):\n","  def __init__(self):\n","    self.word2idx= {}\n","    self.idx2word= {}\n","    self.idx= 0\n","  \n","  def add_word(self,word):\n","    if word not in self.word2idx:\n","      self.word2idx[word]=self.idx\n","      self.idx2word[self.idx]=word\n","      self.idx+=1\n","      \n","  def __len__(self):\n","    return len(self.word2idx)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"lIJR74iV06Ld","colab_type":"code","colab":{}},"source":["class TextProcess(object):\n","  def __init__(self):\n","    self.dictionary=Dictionary()\n","    \n","  def get_data(self, path, batch_size=20):\n","    with open(path, 'r') as f:\n","      tokens=0\n","      for line in f:\n","        words=line.split()+['<eos>']\n","        tokens +=len(words)\n","        for word in words:\n","          self.dictionary.add_word(word)\n","    # representation tensor = rep_tensor\n","    rep_tensor=torch.LongTensor(tokens)\n","    index=0\n","    with open(path, 'r') as f:\n","      for line in f:\n","        words=line.split()+['<eos>']\n","        for word in words:\n","          rep_tensor[index]=self.dictionary.word2idx[word]\n","          index+=1\n","    \n","    num_batches = rep_tensor.shape[0] // batch_size\n","    rep_tensor= rep_tensor[:num_batches*batch_size]\n","    rep_tensor = rep_tensor.view(batch_size,-1)\n","    \n","    return rep_tensor\n","    "],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"CHU3aeq46zGS","colab_type":"code","colab":{}},"source":["embed_size =128 # input size to the LSTM\n","hidden_size = 1024 # number of LSTM unit\n","num_layers = 1\n","num_epochs= 20\n","batch_size= 20\n","timesteps= 30\n","learning_rate= 0.002"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"TZ5hllA2Spms","colab_type":"code","colab":{}},"source":["corpus=TextProcess()\n","rep_tensor=corpus.get_data('drive/Fast-Pytorch/Learning_Pytorch/datasets/example_ebook.txt', batch_size)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"sEeXx25DUqcZ","colab_type":"code","outputId":"afe89403-73cf-4ebd-f660-43516c1d5e88","executionInfo":{"status":"ok","timestamp":1557214213858,"user_tz":-180,"elapsed":691,"user":{"displayName":"Omer Sezer","photoUrl":"","userId":"08295833296445258983"}},"colab":{"base_uri":"https://localhost:8080/","height":35}},"source":["print(rep_tensor.shape)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["torch.Size([20, 1488])\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"JgQUQ9mTWyKm","colab_type":"code","outputId":"86c7a6b7-496d-42bc-e5a9-e635546922e4","executionInfo":{"status":"ok","timestamp":1557214215651,"user_tz":-180,"elapsed":541,"user":{"displayName":"Omer Sezer","photoUrl":"","userId":"08295833296445258983"}},"colab":{"base_uri":"https://localhost:8080/","height":35}},"source":["vocab_size=len(corpus.dictionary)\n","print(vocab_size)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["5291\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"EJro_PEiXJwq","colab_type":"code","outputId":"2c2c8444-7fdf-4a04-cc71-95a219d46635","executionInfo":{"status":"ok","timestamp":1557214218121,"user_tz":-180,"elapsed":639,"user":{"displayName":"Omer Sezer","photoUrl":"","userId":"08295833296445258983"}},"colab":{"base_uri":"https://localhost:8080/","height":35}},"source":["num_batches = rep_tensor.shape[1] // timesteps\n","print(num_batches)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["49\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"5dt25HSDd8Y4","colab_type":"code","colab":{}},"source":["class TextGenerator(nn.Module):\n","  def __init__(self, vocab_size, embed_size, hidden_size, num_layers):\n","    super(TextGenerator, self).__init__()\n","    self.embed= nn.Embedding(vocab_size,embed_size)\n","    self.lstm=nn.LSTM(embed_size,hidden_size,num_layers, batch_first=True)\n","    self.linear=nn.Linear(hidden_size, vocab_size)\n","    \n","  def forward(self,x,h):\n","    x= self.embed(x)\n","    # h: hidden_state, c=output\n","    # x= x.view(batch_size,timesteps,embed_size)\n","    out, (h,c)=self.lstm(x,h)\n","    #(batch_size*timesteps, hidden_size)\n","    #out.size(0):batch_size; out.size(1):timesteps, out.size(2): hidden_size\n","    out=out.reshape(out.size(0)*out.size(1),out.size(2))\n","    # decode hidden states of all time steps\n","    out= self.linear(out)\n","    return out, (h,c)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"kGz-NEJfDL1F","colab_type":"code","colab":{}},"source":["model = TextGenerator(vocab_size, embed_size, hidden_size, num_layers)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"XkcOVcy_DWhG","colab_type":"code","colab":{}},"source":["loss_fn=nn.CrossEntropyLoss()\n","optimizer= torch.optim.Adam( model.parameters(), lr=learning_rate)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"i16KrtuZDicT","colab_type":"code","outputId":"09b122c7-4fda-42f3-c7b0-15f01f40b044","executionInfo":{"status":"ok","timestamp":1557215849360,"user_tz":-180,"elapsed":1609325,"user":{"displayName":"Omer Sezer","photoUrl":"","userId":"08295833296445258983"}},"colab":{"base_uri":"https://localhost:8080/","height":403}},"source":["# Train Model\n","for epoch in range(num_epochs):\n","  # Set initial hidden and cell states\n","  states=(torch.zeros(num_layers, batch_size, hidden_size), \n","          torch.zeros(num_layers, batch_size, hidden_size))\n","  \n","  for i in range(0, rep_tensor.size(1)-timesteps, timesteps):\n","    \n","    inputs=rep_tensor[:,i:i+timesteps]\n","    targets=rep_tensor[:,(i+1):(i+1)+timesteps]\n","    \n","    outputs,_ =model(inputs, states)\n","    loss = loss_fn(outputs, targets.reshape(-1))\n","    \n","    model.zero_grad()\n","    loss.backward()\n","    \n","    clip_grad_norm(model.parameters(),0.5)\n","    optimizer.step()\n","    \n","    step= (i+1) // timesteps\n","    if step % 100 ==0:\n","      print(\"Epoch [{}/{}], Loss: {:.4f}\".format(epoch+1, num_epochs, loss.item()))    "],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:17: UserWarning: torch.nn.utils.clip_grad_norm is now deprecated in favor of torch.nn.utils.clip_grad_norm_.\n"],"name":"stderr"},{"output_type":"stream","text":["Epoch [1/20], Loss: 8.5742\n","Epoch [2/20], Loss: 6.0512\n","Epoch [3/20], Loss: 5.2896\n","Epoch [4/20], Loss: 4.7067\n","Epoch [5/20], Loss: 4.2426\n","Epoch [6/20], Loss: 3.7330\n","Epoch [7/20], Loss: 3.3553\n","Epoch [8/20], Loss: 2.9581\n","Epoch [9/20], Loss: 2.5796\n","Epoch [10/20], Loss: 2.2565\n","Epoch [11/20], Loss: 1.9790\n","Epoch [12/20], Loss: 1.7229\n","Epoch [13/20], Loss: 1.4374\n","Epoch [14/20], Loss: 1.2595\n","Epoch [15/20], Loss: 1.0084\n","Epoch [16/20], Loss: 0.7778\n","Epoch [17/20], Loss: 0.5328\n","Epoch [18/20], Loss: 0.3433\n","Epoch [19/20], Loss: 0.2120\n","Epoch [20/20], Loss: 0.1462\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"hKADkaNQE58v","colab_type":"code","outputId":"5757327b-4fd6-42d4-d915-e2e670f6fca5","executionInfo":{"status":"ok","timestamp":1557216478936,"user_tz":-180,"elapsed":7431,"user":{"displayName":"Omer Sezer","photoUrl":"","userId":"08295833296445258983"}},"colab":{"base_uri":"https://localhost:8080/","height":18476}},"source":["# Test Model\n","with torch.no_grad():\n","  with open(\"drive/Fast-Pytorch/Learning_Pytorch/results/results-RNN-word-embeddings.txt\", \"w\") as f:\n","    state=(torch.zeros(num_layers,1,hidden_size),\n","          torch.zeros(num_layers,1,hidden_size))\n","    \n","    input=torch.randint(0,vocab_size,(1,)).long().unsqueeze(1)\n","    \n","    for i in range(500):\n","      output, _=model(input,state)\n","      print(output.shape)\n","      prob=output.exp()\n","      word_id=torch.multinomial(prob, num_samples=1).item()\n","      print(word_id)\n","      input.fill_(word_id)\n","      \n","      word=corpus.dictionary.idx2word[word_id]\n","      word='\\n' if word==\"<eos>\" else word + ' '\n","      f.write(word)\n","      \n","      if(i+1)%100==0:\n","        print(\"Sampled [{}/{}] words and save to {}\".format(i+1, 500, 'drive/Fast-Pytorch/Learning_Pytorch/results/results-RNN-word-embeddings.txt'))  \n","      "],"execution_count":0,"outputs":[{"output_type":"stream","text":["torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","3969\n","torch.Size([1, 5291])\n","13\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","4413\n","torch.Size([1, 5291])\n","4142\n","torch.Size([1, 5291])\n","55\n","torch.Size([1, 5291])\n","27\n","torch.Size([1, 5291])\n","177\n","torch.Size([1, 5291])\n","3677\n","torch.Size([1, 5291])\n","20\n","torch.Size([1, 5291])\n","489\n","torch.Size([1, 5291])\n","571\n","torch.Size([1, 5291])\n","1308\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","9\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","78\n","torch.Size([1, 5291])\n","110\n","torch.Size([1, 5291])\n","666\n","torch.Size([1, 5291])\n","270\n","torch.Size([1, 5291])\n","202\n","torch.Size([1, 5291])\n","491\n","torch.Size([1, 5291])\n","110\n","torch.Size([1, 5291])\n","251\n","torch.Size([1, 5291])\n","34\n","torch.Size([1, 5291])\n","334\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","2652\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","3437\n","torch.Size([1, 5291])\n","38\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","501\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","362\n","torch.Size([1, 5291])\n","10\n","torch.Size([1, 5291])\n","95\n","torch.Size([1, 5291])\n","16\n","torch.Size([1, 5291])\n","2638\n","torch.Size([1, 5291])\n","2544\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","150\n","torch.Size([1, 5291])\n","188\n","torch.Size([1, 5291])\n","18\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","167\n","torch.Size([1, 5291])\n","9\n","torch.Size([1, 5291])\n","151\n","torch.Size([1, 5291])\n","238\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","4009\n","torch.Size([1, 5291])\n","4169\n","torch.Size([1, 5291])\n","68\n","torch.Size([1, 5291])\n","1330\n","torch.Size([1, 5291])\n","3125\n","torch.Size([1, 5291])\n","13\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","931\n","torch.Size([1, 5291])\n","936\n","torch.Size([1, 5291])\n","2983\n","torch.Size([1, 5291])\n","103\n","torch.Size([1, 5291])\n","788\n","torch.Size([1, 5291])\n","73\n","torch.Size([1, 5291])\n","759\n","torch.Size([1, 5291])\n","1380\n","torch.Size([1, 5291])\n","822\n","torch.Size([1, 5291])\n","610\n","torch.Size([1, 5291])\n","28\n","torch.Size([1, 5291])\n","1963\n","torch.Size([1, 5291])\n","34\n","torch.Size([1, 5291])\n","77\n","torch.Size([1, 5291])\n","2165\n","torch.Size([1, 5291])\n","931\n","torch.Size([1, 5291])\n","1064\n","torch.Size([1, 5291])\n","272\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","78\n","torch.Size([1, 5291])\n","79\n","torch.Size([1, 5291])\n","7\n","torch.Size([1, 5291])\n","917\n","torch.Size([1, 5291])\n","1197\n","torch.Size([1, 5291])\n","4561\n","torch.Size([1, 5291])\n","52\n","torch.Size([1, 5291])\n","174\n","torch.Size([1, 5291])\n","1757\n","torch.Size([1, 5291])\n","38\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","2922\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","3008\n","Sampled [100/500] words and save to drive/Colab/Learning_Pytorch/results/results-RNN-word-embeddings.txt\n","torch.Size([1, 5291])\n","931\n","torch.Size([1, 5291])\n","1625\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","877\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","675\n","torch.Size([1, 5291])\n","38\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","1201\n","torch.Size([1, 5291])\n","1249\n","torch.Size([1, 5291])\n","20\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","295\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","537\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","2835\n","torch.Size([1, 5291])\n","989\n","torch.Size([1, 5291])\n","34\n","torch.Size([1, 5291])\n","1968\n","torch.Size([1, 5291])\n","25\n","torch.Size([1, 5291])\n","640\n","torch.Size([1, 5291])\n","94\n","torch.Size([1, 5291])\n","114\n","torch.Size([1, 5291])\n","245\n","torch.Size([1, 5291])\n","280\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","38\n","torch.Size([1, 5291])\n","3299\n","torch.Size([1, 5291])\n","4208\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","668\n","torch.Size([1, 5291])\n","668\n","torch.Size([1, 5291])\n","668\n","torch.Size([1, 5291])\n","668\n","torch.Size([1, 5291])\n","668\n","torch.Size([1, 5291])\n","668\n","torch.Size([1, 5291])\n","668\n","torch.Size([1, 5291])\n","668\n","torch.Size([1, 5291])\n","668\n","torch.Size([1, 5291])\n","668\n","torch.Size([1, 5291])\n","668\n","torch.Size([1, 5291])\n","668\n","torch.Size([1, 5291])\n","668\n","torch.Size([1, 5291])\n","668\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","3400\n","torch.Size([1, 5291])\n","3401\n","torch.Size([1, 5291])\n","822\n","torch.Size([1, 5291])\n","13\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","4009\n","torch.Size([1, 5291])\n","4169\n","torch.Size([1, 5291])\n","68\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","3008\n","torch.Size([1, 5291])\n","20\n","torch.Size([1, 5291])\n","1373\n","torch.Size([1, 5291])\n","701\n","torch.Size([1, 5291])\n","4851\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","114\n","torch.Size([1, 5291])\n","16\n","torch.Size([1, 5291])\n","550\n","torch.Size([1, 5291])\n","9\n","torch.Size([1, 5291])\n","809\n","torch.Size([1, 5291])\n","1304\n","torch.Size([1, 5291])\n","272\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","1216\n","torch.Size([1, 5291])\n","95\n","torch.Size([1, 5291])\n","16\n","torch.Size([1, 5291])\n","3804\n","torch.Size([1, 5291])\n","9\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","20\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","3008\n","torch.Size([1, 5291])\n","110\n","torch.Size([1, 5291])\n","7\n","torch.Size([1, 5291])\n","22\n","torch.Size([1, 5291])\n","33\n","torch.Size([1, 5291])\n","3865\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","6\n","torch.Size([1, 5291])\n","7\n","torch.Size([1, 5291])\n","173\n","torch.Size([1, 5291])\n","272\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","4413\n","torch.Size([1, 5291])\n","38\n","torch.Size([1, 5291])\n","394\n","torch.Size([1, 5291])\n","1164\n","torch.Size([1, 5291])\n","1509\n","Sampled [200/500] words and save to drive/Colab/Learning_Pytorch/results/results-RNN-word-embeddings.txt\n","torch.Size([1, 5291])\n","73\n","torch.Size([1, 5291])\n","11\n","torch.Size([1, 5291])\n","12\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","1386\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","780\n","torch.Size([1, 5291])\n","7\n","torch.Size([1, 5291])\n","705\n","torch.Size([1, 5291])\n","4846\n","torch.Size([1, 5291])\n","3207\n","torch.Size([1, 5291])\n","114\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","160\n","torch.Size([1, 5291])\n","202\n","torch.Size([1, 5291])\n","203\n","torch.Size([1, 5291])\n","3547\n","torch.Size([1, 5291])\n","20\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","3162\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","514\n","torch.Size([1, 5291])\n","38\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","4453\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","42\n","torch.Size([1, 5291])\n","13\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","160\n","torch.Size([1, 5291])\n","6\n","torch.Size([1, 5291])\n","7\n","torch.Size([1, 5291])\n","2912\n","torch.Size([1, 5291])\n","415\n","torch.Size([1, 5291])\n","1524\n","torch.Size([1, 5291])\n","5119\n","torch.Size([1, 5291])\n","843\n","torch.Size([1, 5291])\n","1903\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","777\n","torch.Size([1, 5291])\n","103\n","torch.Size([1, 5291])\n","1189\n","torch.Size([1, 5291])\n","103\n","torch.Size([1, 5291])\n","788\n","torch.Size([1, 5291])\n","1002\n","torch.Size([1, 5291])\n","3474\n","torch.Size([1, 5291])\n","38\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","11\n","torch.Size([1, 5291])\n","1648\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","575\n","torch.Size([1, 5291])\n","13\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5132\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","0\n","torch.Size([1, 5291])\n","1\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","59\n","torch.Size([1, 5291])\n","2013\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","11\n","torch.Size([1, 5291])\n","12\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","160\n","torch.Size([1, 5291])\n","202\n","torch.Size([1, 5291])\n","491\n","torch.Size([1, 5291])\n","176\n","torch.Size([1, 5291])\n","38\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","179\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","8\n","torch.Size([1, 5291])\n","471\n","torch.Size([1, 5291])\n","267\n","torch.Size([1, 5291])\n","7\n","torch.Size([1, 5291])\n","2912\n","torch.Size([1, 5291])\n","4287\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","0\n","torch.Size([1, 5291])\n","1\n","torch.Size([1, 5291])\n","2\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","919\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","3415\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","2065\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","3055\n","torch.Size([1, 5291])\n","203\n","torch.Size([1, 5291])\n","2695\n","Sampled [300/500] words and save to drive/Colab/Learning_Pytorch/results/results-RNN-word-embeddings.txt\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","160\n","torch.Size([1, 5291])\n","6\n","torch.Size([1, 5291])\n","3722\n","torch.Size([1, 5291])\n","6\n","torch.Size([1, 5291])\n","38\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","154\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","303\n","torch.Size([1, 5291])\n","4186\n","torch.Size([1, 5291])\n","601\n","torch.Size([1, 5291])\n","1577\n","torch.Size([1, 5291])\n","103\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","20\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","4413\n","torch.Size([1, 5291])\n","15\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","78\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","78\n","torch.Size([1, 5291])\n","79\n","torch.Size([1, 5291])\n","7\n","torch.Size([1, 5291])\n","173\n","torch.Size([1, 5291])\n","112\n","torch.Size([1, 5291])\n","4798\n","torch.Size([1, 5291])\n","2456\n","torch.Size([1, 5291])\n","4563\n","torch.Size([1, 5291])\n","80\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","501\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","2922\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","3047\n","torch.Size([1, 5291])\n","251\n","torch.Size([1, 5291])\n","34\n","torch.Size([1, 5291])\n","248\n","torch.Size([1, 5291])\n","69\n","torch.Size([1, 5291])\n","281\n","torch.Size([1, 5291])\n","95\n","torch.Size([1, 5291])\n","16\n","torch.Size([1, 5291])\n","550\n","torch.Size([1, 5291])\n","9\n","torch.Size([1, 5291])\n","809\n","torch.Size([1, 5291])\n","1716\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","1960\n","torch.Size([1, 5291])\n","394\n","torch.Size([1, 5291])\n","1164\n","torch.Size([1, 5291])\n","4204\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","49\n","torch.Size([1, 5291])\n","202\n","torch.Size([1, 5291])\n","491\n","torch.Size([1, 5291])\n","28\n","torch.Size([1, 5291])\n","1570\n","torch.Size([1, 5291])\n","20\n","torch.Size([1, 5291])\n","288\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","705\n","torch.Size([1, 5291])\n","4846\n","torch.Size([1, 5291])\n","4798\n","torch.Size([1, 5291])\n","2283\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","20\n","torch.Size([1, 5291])\n","2828\n","torch.Size([1, 5291])\n","25\n","torch.Size([1, 5291])\n","640\n","torch.Size([1, 5291])\n","94\n","torch.Size([1, 5291])\n","114\n","torch.Size([1, 5291])\n","3932\n","torch.Size([1, 5291])\n","46\n","torch.Size([1, 5291])\n","672\n","torch.Size([1, 5291])\n","46\n","torch.Size([1, 5291])\n","202\n","torch.Size([1, 5291])\n","203\n","torch.Size([1, 5291])\n","3547\n","torch.Size([1, 5291])\n","9\n","torch.Size([1, 5291])\n","151\n","torch.Size([1, 5291])\n","41\n","torch.Size([1, 5291])\n","34\n","torch.Size([1, 5291])\n","463\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","1168\n","torch.Size([1, 5291])\n","285\n","torch.Size([1, 5291])\n","2024\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","256\n","torch.Size([1, 5291])\n","251\n","Sampled [400/500] words and save to drive/Colab/Learning_Pytorch/results/results-RNN-word-embeddings.txt\n","torch.Size([1, 5291])\n","34\n","torch.Size([1, 5291])\n","248\n","torch.Size([1, 5291])\n","69\n","torch.Size([1, 5291])\n","281\n","torch.Size([1, 5291])\n","95\n","torch.Size([1, 5291])\n","16\n","torch.Size([1, 5291])\n","390\n","torch.Size([1, 5291])\n","13\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","362\n","torch.Size([1, 5291])\n","151\n","torch.Size([1, 5291])\n","34\n","torch.Size([1, 5291])\n","55\n","torch.Size([1, 5291])\n","27\n","torch.Size([1, 5291])\n","3452\n","torch.Size([1, 5291])\n","9\n","torch.Size([1, 5291])\n","6\n","torch.Size([1, 5291])\n","3722\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","295\n","torch.Size([1, 5291])\n","57\n","torch.Size([1, 5291])\n","169\n","torch.Size([1, 5291])\n","3108\n","torch.Size([1, 5291])\n","150\n","torch.Size([1, 5291])\n","188\n","torch.Size([1, 5291])\n","18\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","4355\n","torch.Size([1, 5291])\n","41\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","532\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","3400\n","torch.Size([1, 5291])\n","68\n","torch.Size([1, 5291])\n","1330\n","torch.Size([1, 5291])\n","3125\n","torch.Size([1, 5291])\n","38\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","2818\n","torch.Size([1, 5291])\n","575\n","torch.Size([1, 5291])\n","13\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","547\n","torch.Size([1, 5291])\n","20\n","torch.Size([1, 5291])\n","11\n","torch.Size([1, 5291])\n","12\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","877\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","877\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","3415\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","33\n","torch.Size([1, 5291])\n","3865\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","780\n","torch.Size([1, 5291])\n","9\n","torch.Size([1, 5291])\n","69\n","torch.Size([1, 5291])\n","281\n","torch.Size([1, 5291])\n","13\n","torch.Size([1, 5291])\n","3505\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","59\n","torch.Size([1, 5291])\n","4067\n","torch.Size([1, 5291])\n","594\n","torch.Size([1, 5291])\n","7\n","torch.Size([1, 5291])\n","22\n","torch.Size([1, 5291])\n","33\n","torch.Size([1, 5291])\n","1380\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","320\n","torch.Size([1, 5291])\n","989\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","160\n","torch.Size([1, 5291])\n","202\n","torch.Size([1, 5291])\n","203\n","torch.Size([1, 5291])\n","2695\n","torch.Size([1, 5291])\n","3\n","torch.Size([1, 5291])\n","4009\n","torch.Size([1, 5291])\n","4169\n","torch.Size([1, 5291])\n","2366\n","torch.Size([1, 5291])\n","562\n","torch.Size([1, 5291])\n","3131\n","torch.Size([1, 5291])\n","44\n","torch.Size([1, 5291])\n","792\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","6\n","torch.Size([1, 5291])\n","9\n","torch.Size([1, 5291])\n","1310\n","torch.Size([1, 5291])\n","38\n","torch.Size([1, 5291])\n","3299\n","torch.Size([1, 5291])\n","1395\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","5\n","torch.Size([1, 5291])\n","148\n","torch.Size([1, 5291])\n","1381\n","torch.Size([1, 5291])\n","1235\n","torch.Size([1, 5291])\n","82\n","Sampled [500/500] words and save to drive/Colab/Learning_Pytorch/results/results-RNN-word-embeddings.txt\n"],"name":"stdout"}]}]}