Make sure methods get expected values and fix tests

This is still kind of messy, I just wanted to make sure methods get the
expected values for now. But I should clean up the fallback method,
and the parse method.

Author: vasconsaurus

app/models/media.rb

@@ -82,12 +82,15 @@ def process_and_return_json(options = {})
     cache = Pender::Store.current
     if options.delete(:force) || cache.read(id, :json).nil?
       handle_exceptions(self, StandardError) { self.parse }
+      clean_data = clean_json(self.data)
+      self.fallback
+
       if data[:error].blank?
-        cache.write(id, :json, clean_json(data))
+        cache.write(id, :json, clean_data)
       end
       self.upload_images
     end
-    self.fallback
+
     archive_if_conditions_are_met(options, id, cache)
     parser_requests_metrics
     cache.read(id, :json) || clean_json(data)
@@ -207,8 +210,14 @@ def parse
           self.doc,
           self.original_url,
           self.data.dig('raw', 'json+ld')
+          )
+        self.data.deep_merge!(
+          {
+            provider: self.provider,
+            type: self.type,
+            url: self.parser.url
+          }.merge(parsed_data)
         )
-        self.data.deep_merge!(parsed_data)
         self.url = self.parser.url
         self.get_oembed_data
         parsed = true

test/helpers/medias_test.rb

@@ -26,7 +26,6 @@ def setup
   test "should not crash if jsonld content is null" do
     null_content = '<script type="application/ld+json">null</script>'
     m = create_media url: 'https://www.facebook.com/dina.samak/posts/10153679232246949'
-    m.data = Media.minimal_data(m)
     Media.any_instance.stubs(:doc).returns(Nokogiri::HTML(null_content))
     assert_nothing_raised do
       get_jsonld_data(m)
@@ -39,7 +38,6 @@ def setup
     doc = ''
     File.open('test/data/page-with-json-ld.html') { |f| doc = f.read }
     Media.any_instance.stubs(:doc).returns(Nokogiri::HTML(doc))
-    m.data = Media.minimal_data(m)
     assert_nothing_raised do
       m.get_jsonld_data(m)
     end

test/integration/parser/facebook_item_test.rb

@@ -36,11 +36,11 @@ class FacebookItemIntegrationTest < ActiveSupport::TestCase
     assert_equal 'item', data['type']
     assert_equal '111111111111111_1111111111111111', data['external_id']
     assert_match(/facebook.com\/111111111111111\/posts\/1111111111111111/, data['title'])
+    assert_match(/facebook.com\/111111111111111\/posts\/1111111111111111/, data['description'])
     assert_equal '', data['username']
     assert_equal '', data['author_name']
     assert_equal '', data['author_picture']
     assert_equal '', data['author_url']
-    assert_equal '', data['description']
     assert_equal '', data['picture']
     assert_equal '', data['published_at']
   end

test/integration/parser/facebook_profile_test.rb

@@ -37,12 +37,11 @@ class FacebookProfileIntegrationTest < ActiveSupport::TestCase
     data = media.process_and_return_json
 
     assert_match(/facebook.com\/pages\/fakepage\/1111111111111/, data['title'])
+    assert_match(/facebook.com\/pages\/fakepage\/1111111111111/, data['description'])
     assert_equal 'fakepage', data['username']
-    assert data['description'].blank?
     assert data['picture'].blank?
     assert data['published_at'].blank?
     assert_equal 'facebook', data['provider']
     assert_equal 'profile', data['type']
   end
 end
-

test/integration/parser/kwai_test.rb

@@ -18,7 +18,7 @@ class KwaiIntegrationTest < ActiveSupport::TestCase
     data = m.process_and_return_json
 
     assert_equal 'https://kwai-video.com/p/aaaaaaaa', data['title']
-    assert data['description'].blank?
+    assert_equal 'https://kwai-video.com/p/aaaaaaaa', data['description']
     assert data['username'].blank?
     assert_equal 'kwai', data['provider']
     assert_equal 'item', data['type']

test/models/media_test.rb

@@ -94,7 +94,7 @@ class MediaTest < ActiveSupport::TestCase
     m = create_media url: 'http://xkcd.com/448/'
     data = m.process_and_return_json
     assert_match /Good Morning/, data['title']
-    assert_equal '', data['description']
+    assert_equal 'https://xkcd.com/448/', data['description']
     assert_equal '', data['published_at']
     assert_equal '', data['username']
     assert_match 'https://xkcd.com', data['author_url']
@@ -226,12 +226,11 @@ class MediaTest < ActiveSupport::TestCase
     doc = ''
     File.open('test/data/page-with-json-ld.html') { |f| doc = f.read }
     Media.any_instance.stubs(:doc).returns(Nokogiri::HTML(doc))
-    m.data = Media.minimal_data(m)
-    m.get_jsonld_data(m)
+    data = m.process_and_return_json
 
-    assert !m.data['raw']['json+ld'].empty?
-    assert m.data['raw']['json+ld'].is_a? Array
-    assert m.data['raw']['json+ld'].first.is_a? Hash
+    assert_not_empty data['raw']['json+ld']
+    assert_kind_of Array, data['raw']['json+ld']
+    assert_kind_of Hash, data['raw']['json+ld'].first
   end
 
   test "should handle errors when call parse on each parser" do

test/models/parser/facebook_item_test.rb

@@ -454,7 +454,7 @@ def apify_content_not_available_response
     assert_match "<div class=\"fb-post\" data-href=\"#{url}\"></div>", data['html']
   end
 
-  test "should return html and empty description when FB url is private" do
+  test "should return html and description with the original url when FB url is private" do
     url = 'https://www.facebook.com/caiosba/posts/1913749825339929'
 
     WebMock.stub_request(:post, /api\.apify\.com\/v2\/acts\/apify/).to_return(status: 200, body: apify_response_not_found)
@@ -471,7 +471,7 @@ def apify_content_not_available_response
     media = Media.new(url: url)
     data = media.process_and_return_json
 
-    assert data[:description].empty?
+    assert_equal url, data[:description]
     assert_match "<div class=\"fb-post\" data-href=\"https://www.facebook.com/caiosba/posts/1913749825339929\">", data['html']
   end