diff --git a/src/json-ld.net/Core/DocumentLoader.cs b/src/json-ld.net/Core/DocumentLoader.cs index 99bde79..9c759ae 100644 --- a/src/json-ld.net/Core/DocumentLoader.cs +++ b/src/json-ld.net/Core/DocumentLoader.cs @@ -6,224 +6,108 @@ using JsonLD.Util; using System.Net; using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading.Tasks; +using System.Runtime.InteropServices; namespace JsonLD.Core { public class DocumentLoader { + enum JsonLDContentType + { + JsonLD, + PlainJson, + Other + } + + JsonLDContentType GetJsonLDContentType(string contentTypeStr) + { + JsonLDContentType contentType; + + switch (contentTypeStr) + { + case "application/ld+json": + contentType = JsonLDContentType.JsonLD; + break; + // From RFC 6839, it looks like plain JSON is content type application/json and any MediaType ending in "+json". + case "application/json": + case string type when type.EndsWith("+json"): + contentType = JsonLDContentType.PlainJson; + break; + default: + contentType = JsonLDContentType.Other; + break; + } + + return contentType; + } + /// public virtual RemoteDocument LoadDocument(string url) { -#if !PORTABLE && !IS_CORECLR - RemoteDocument doc = new RemoteDocument(url, null); - HttpWebResponse resp; + return LoadDocumentAsync(url).ConfigureAwait(false).GetAwaiter().GetResult(); + } + /// + public virtual async Task LoadDocumentAsync(string url) + { + RemoteDocument doc = new RemoteDocument(url, null); try { - HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(url); - req.Accept = _acceptHeader; - resp = (HttpWebResponse)req.GetResponse(); - bool isJsonld = resp.Headers[HttpResponseHeader.ContentType] == "application/ld+json"; - if (!resp.Headers[HttpResponseHeader.ContentType].Contains("json")) + using (HttpResponseMessage response = await JsonLD.Util.LDHttpClient.FetchAsync(url).ConfigureAwait(false)) { - throw new JsonLdError(JsonLdError.Error.LoadingDocumentFailed, url); - } - string[] linkHeaders = resp.Headers.GetValues("Link"); - if (!isJsonld && linkHeaders != null) - { - linkHeaders = linkHeaders.SelectMany((h) => h.Split(",".ToCharArray())) - .Select(h => h.Trim()).ToArray(); - IEnumerable linkedContexts = linkHeaders.Where(v => v.EndsWith("rel=\"http://www.w3.org/ns/json-ld#context\"")); - if (linkedContexts.Count() > 1) + var code = (int)response.StatusCode; + + if (code >= 400) { - throw new JsonLdError(JsonLdError.Error.MultipleContextLinkHeaders); + throw new JsonLdError(JsonLdError.Error.LoadingDocumentFailed, $"HTTP {code} {url}"); } - string header = linkedContexts.First(); - string linkedUrl = header.Substring(1, header.IndexOf(">") - 1); - string resolvedUrl = URL.Resolve(url, linkedUrl); - var remoteContext = this.LoadDocument(resolvedUrl); - doc.contextUrl = remoteContext.documentUrl; - doc.context = remoteContext.document; - } - Stream stream = resp.GetResponseStream(); + var finalUrl = response.RequestMessage.RequestUri.ToString(); - doc.DocumentUrl = req.Address.ToString(); - doc.Document = JSONUtils.FromInputStream(stream); - } - catch (JsonLdError) - { - throw; - } - catch (WebException webException) - { - try - { - resp = (HttpWebResponse)webException.Response; - int baseStatusCode = (int)(Math.Floor((double)resp.StatusCode / 100)) * 100; - if (baseStatusCode == 300) + var contentType = GetJsonLDContentType(response.Content.Headers.ContentType.MediaType); + + if (contentType == JsonLDContentType.Other) { - string location = resp.Headers[HttpResponseHeader.Location]; - if (!string.IsNullOrWhiteSpace(location)) + throw new JsonLdError(JsonLdError.Error.LoadingDocumentFailed, url); + } + + // For plain JSON, see if there's a context document linked in the HTTP response headers. + if (contentType == JsonLDContentType.PlainJson && response.Headers.TryGetValues("Link", out var linkHeaders)) + { + linkHeaders = linkHeaders.SelectMany((h) => h.Split(",".ToCharArray())) + .Select(h => h.Trim()).ToArray(); + IEnumerable linkedContexts = linkHeaders.Where(v => v.EndsWith("rel=\"http://www.w3.org/ns/json-ld#context\"")); + if (linkedContexts.Count() > 1) { - // TODO: Add recursion break or simply switch to HttpClient so we don't have to recurse on HTTP redirects. - return LoadDocument(location); + throw new JsonLdError(JsonLdError.Error.MultipleContextLinkHeaders); } + string header = linkedContexts.First(); + string linkedUrl = header.Substring(1, header.IndexOf(">") - 1); + string resolvedUrl = URL.Resolve(finalUrl, linkedUrl); + var remoteContext = await this.LoadDocumentAsync(resolvedUrl).ConfigureAwait(false); + doc.contextUrl = remoteContext.documentUrl; + doc.context = remoteContext.document; } - } - catch (Exception innerException) - { - throw new JsonLdError(JsonLdError.Error.LoadingDocumentFailed, url, innerException); - } - throw new JsonLdError(JsonLdError.Error.LoadingDocumentFailed, url, webException); + Stream stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); + + doc.DocumentUrl = finalUrl; + doc.Document = JSONUtils.FromInputStream(stream); + } + } + catch (JsonLdError) + { + throw; } catch (Exception exception) { throw new JsonLdError(JsonLdError.Error.LoadingDocumentFailed, url, exception); } return doc; -#else - throw new PlatformNotSupportedException(); -#endif } - - /// An HTTP Accept header that prefers JSONLD. - /// An HTTP Accept header that prefers JSONLD. - private const string _acceptHeader = "application/ld+json, application/json;q=0.9, application/javascript;q=0.5, text/javascript;q=0.5, text/plain;q=0.2, */*;q=0.1"; - -// private static volatile IHttpClient httpClient; - -// /// -// /// Returns a Map, List, or String containing the contents of the JSON -// /// resource resolved from the URL. -// /// -// /// -// /// Returns a Map, List, or String containing the contents of the JSON -// /// resource resolved from the URL. -// /// -// /// The URL to resolve -// /// -// /// The Map, List, or String that represent the JSON resource -// /// resolved from the URL -// /// -// /// If the JSON was not valid. -// /// -// /// If there was an error resolving the resource. -// /// -// public static object FromURL(URL url) -// { -// MappingJsonFactory jsonFactory = new MappingJsonFactory(); -// InputStream @in = OpenStreamFromURL(url); -// try -// { -// JsonParser parser = jsonFactory.CreateParser(@in); -// try -// { -// JsonToken token = parser.NextToken(); -// Type type; -// if (token == JsonToken.StartObject) -// { -// type = typeof(IDictionary); -// } -// else -// { -// if (token == JsonToken.StartArray) -// { -// type = typeof(IList); -// } -// else -// { -// type = typeof(string); -// } -// } -// return parser.ReadValueAs(type); -// } -// finally -// { -// parser.Close(); -// } -// } -// finally -// { -// @in.Close(); -// } -// } - -// /// -// /// Opens an -// /// Java.IO.InputStream -// /// for the given -// /// Java.Net.URL -// /// , including support -// /// for http and https URLs that are requested using Content Negotiation with -// /// application/ld+json as the preferred content type. -// /// -// /// The URL identifying the source. -// /// An InputStream containing the contents of the source. -// /// If there was an error resolving the URL. -// public static InputStream OpenStreamFromURL(URL url) -// { -// string protocol = url.GetProtocol(); -// if (!JsonLDNet.Shims.EqualsIgnoreCase(protocol, "http") && !JsonLDNet.Shims.EqualsIgnoreCase -// (protocol, "https")) -// { -// // Can't use the HTTP client for those! -// // Fallback to Java's built-in URL handler. No need for -// // Accept headers as it's likely to be file: or jar: -// return url.OpenStream(); -// } -// IHttpUriRequest request = new HttpGet(url.ToExternalForm()); -// // We prefer application/ld+json, but fallback to application/json -// // or whatever is available -// request.AddHeader("Accept", AcceptHeader); -// IHttpResponse response = GetHttpClient().Execute(request); -// int status = response.GetStatusLine().GetStatusCode(); -// if (status != 200 && status != 203) -// { -// throw new IOException("Can't retrieve " + url + ", status code: " + status); -// } -// return response.GetEntity().GetContent(); -// } - -// public static IHttpClient GetHttpClient() -// { -// IHttpClient result = httpClient; -// if (result == null) -// { -// lock (typeof(JSONUtils)) -// { -// result = httpClient; -// if (result == null) -// { -// // Uses Apache SystemDefaultHttpClient rather than -// // DefaultHttpClient, thus the normal proxy settings for the -// // JVM will be used -// DefaultHttpClient client = new SystemDefaultHttpClient(); -// // Support compressed data -// // http://hc.apache.org/httpcomponents-client-ga/tutorial/html/httpagent.html#d5e1238 -// client.AddRequestInterceptor(new RequestAcceptEncoding()); -// client.AddResponseInterceptor(new ResponseContentEncoding()); -// CacheConfig cacheConfig = new CacheConfig(); -// cacheConfig.SetMaxObjectSize(1024 * 128); -// // 128 kB -// cacheConfig.SetMaxCacheEntries(1000); -// // and allow caching -// httpClient = new CachingHttpClient(client, cacheConfig); -// result = httpClient; -// } -// } -// } -// return result; -// } - -// public static void SetHttpClient(IHttpClient nextHttpClient) -// { -// lock (typeof(JSONUtils)) -// { -// httpClient = nextHttpClient; -// } -// } } } diff --git a/src/json-ld.net/Util/JSONUtils.cs b/src/json-ld.net/Util/JSONUtils.cs index 37a3510..ae13c2d 100644 --- a/src/json-ld.net/Util/JSONUtils.cs +++ b/src/json-ld.net/Util/JSONUtils.cs @@ -1,10 +1,13 @@ using System; using System.Collections; using System.IO; +using System.Linq; using JsonLD.Util; using Newtonsoft.Json; using System.Net; using Newtonsoft.Json.Linq; +using System.Net.Http; +using System.Threading.Tasks; namespace JsonLD.Util { @@ -12,28 +15,10 @@ namespace JsonLD.Util /// tristan internal class JSONUtils { - /// An HTTP Accept header that prefers JSONLD. - /// An HTTP Accept header that prefers JSONLD. - protected internal const string AcceptHeader = "application/ld+json, application/json;q=0.9, application/javascript;q=0.5, text/javascript;q=0.5, text/plain;q=0.2, */*;q=0.1"; - - //private static readonly ObjectMapper JsonMapper = new ObjectMapper(); - - //private static readonly JsonFactory JsonFactory = new JsonFactory(JsonMapper); - static JSONUtils() { - // Disable default Jackson behaviour to close - // InputStreams/Readers/OutputStreams/Writers - //JsonFactory.Disable(JsonGenerator.Feature.AutoCloseTarget); - // Disable string retention features that may work for most JSON where - // the field names are in limited supply, but does not work for JSON-LD - // where a wide range of URIs are used for subjects and predicates - //JsonFactory.Disable(JsonFactory.Feature.InternFieldNames); - //JsonFactory.Disable(JsonFactory.Feature.CanonicalizeFieldNames); } - // private static volatile IHttpClient httpClient; - /// /// public static JToken FromString(string jsonString) @@ -130,6 +115,11 @@ public static string ToString(JToken obj) return sw.ToString(); } + public static JToken FromURL(Uri url) + { + return FromURLAsync(url).ConfigureAwait(false).GetAwaiter().GetResult(); + } + /// /// Returns a Map, List, or String containing the contents of the JSON /// resource resolved from the URL. @@ -147,17 +137,11 @@ public static string ToString(JToken obj) /// /// If there was an error resolving the resource. /// - public static JToken FromURL(Uri url) + public static async Task FromURLAsync(Uri url) { -#if !PORTABLE && !IS_CORECLR - HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(url); - req.Accept = AcceptHeader; - WebResponse resp = req.GetResponse(); - Stream stream = resp.GetResponseStream(); - return FromInputStream(stream); -#else - throw new PlatformNotSupportedException(); -#endif + using (var response = await LDHttpClient.FetchAsync(url.ToString()).ConfigureAwait(false)) { + return FromInputStream(await response.Content.ReadAsStreamAsync().ConfigureAwait(false)); + } } } } diff --git a/src/json-ld.net/Util/LDHttpClient.cs b/src/json-ld.net/Util/LDHttpClient.cs new file mode 100644 index 0000000..170faa9 --- /dev/null +++ b/src/json-ld.net/Util/LDHttpClient.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; + +namespace JsonLD.Util +{ + internal static class LDHttpClient + { + const string ACCEPT_HEADER = "application/ld+json, application/json;q=0.9, application/javascript;q=0.5, text/javascript;q=0.5, text/plain;q=0.2, */*;q=0.1"; + const int MAX_REDIRECTS = 20; + + static HttpClient _hc; + + static LDHttpClient() + { + _hc = new HttpClient(); + _hc.DefaultRequestHeaders.Add("Accept", ACCEPT_HEADER); + } + + static public async Task FetchAsync(string url) + { + int redirects = 0; + int code; + string redirectedUrl = url; + + HttpResponseMessage response; + + // Manually follow redirects because .NET Core refuses to auto-follow HTTPS->HTTP redirects. + do + { + HttpRequestMessage httpRequestMessage = new HttpRequestMessage(HttpMethod.Get, redirectedUrl); + response = await _hc.SendAsync(httpRequestMessage).ConfigureAwait(false); + if (response.Headers.TryGetValues("Location", out var location)) + { + redirectedUrl = location.First(); + } + + code = (int)response.StatusCode; + } while (redirects++ < MAX_REDIRECTS && code >= 300 && code < 400); + + if (redirects >= MAX_REDIRECTS) + { + throw new HttpRequestException("Too many redirects"); + } + + return response; + } + } +} diff --git a/test/json-ld.net.tests/ConformanceTests.cs b/test/json-ld.net.tests/ConformanceTests.cs index 09bcaf8..8604765 100644 --- a/test/json-ld.net.tests/ConformanceTests.cs +++ b/test/json-ld.net.tests/ConformanceTests.cs @@ -60,11 +60,8 @@ public class ConformanceCases: IEnumerable "toRdf-manifest.jsonld", "fromRdf-manifest.jsonld", "normalize-manifest.jsonld", -// Test tests are not supported on CORE CLR -#if !PORTABLE && !IS_CORECLR "error-manifest.jsonld", "remote-doc-manifest.jsonld", -#endif }; public ConformanceCases() @@ -80,13 +77,18 @@ public IEnumerator GetEnumerator() foreach (string manifest in manifests) { JToken manifestJson = jsonFetcher.GetJson(manifest, rootDirectory); + var isRemoteTest = (string)manifestJson["name"] == "Remote document"; foreach (JObject testcase in manifestJson["sequence"]) { Func run; ConformanceCase newCase = new ConformanceCase(); - newCase.input = jsonFetcher.GetJson(testcase["input"], rootDirectory); + // Load input file if not remote test. Remote tests load from the web at test execution time. + if (!isRemoteTest) + { + newCase.input = jsonFetcher.GetJson(testcase["input"], rootDirectory); + } newCase.context = jsonFetcher.GetJson(testcase["context"], rootDirectory); newCase.frame = jsonFetcher.GetJson(testcase["frame"], rootDirectory); @@ -192,7 +194,7 @@ public IEnumerator GetEnumerator() run = () => { throw new Exception("Couldn't find a test type, apparently."); }; } - if ((string)manifestJson["name"] == "Remote document") + if (isRemoteTest) { Func innerRun = run; run = () => diff --git a/test/json-ld.net.tests/JsonFetcher.cs b/test/json-ld.net.tests/JsonFetcher.cs index 35ec309..6e0ece0 100644 --- a/test/json-ld.net.tests/JsonFetcher.cs +++ b/test/json-ld.net.tests/JsonFetcher.cs @@ -22,8 +22,8 @@ public JToken GetJson(JToken j, string rootDirectory) return JToken.ReadFrom(jreader); } } - catch (Exception e) - { // TODO: this should not be here, figure out why this is needed or catch specific exception. + catch (JsonReaderException) + { return null; } } diff --git a/test/json-ld.net.tests/NQuadsParserTests.cs b/test/json-ld.net.tests/NQuadsParserTests.cs index 860db43..5915d22 100644 --- a/test/json-ld.net.tests/NQuadsParserTests.cs +++ b/test/json-ld.net.tests/NQuadsParserTests.cs @@ -35,7 +35,7 @@ public NQuadsParserTests() } [Theory] - [MemberData("PositiveTestCases")] + [MemberData(nameof(PositiveTestCases))] public void PositiveParseTest(string path) { // given @@ -46,7 +46,7 @@ public void PositiveParseTest(string path) } [Theory] - [MemberData("NegativeTestCases")] + [MemberData(nameof(NegativeTestCases))] public void NegativeParseTest(string path) { // given