From 6b56ebb8d86c797728ca9cbe05277794dbb6e66a Mon Sep 17 00:00:00 2001 From: Philippe Duval Date: Tue, 30 Oct 2018 17:05:06 -0400 Subject: [PATCH 01/10] Add integrity hash utils functions. --- dash/_utils.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/dash/_utils.py b/dash/_utils.py index 17dc247ebe..099995e4c9 100644 --- a/dash/_utils.py +++ b/dash/_utils.py @@ -1,3 +1,9 @@ +import functools +import hashlib +import base64 +import pkgutil + + def interpolate_str(template, **data): s = template for k, v in data.items(): @@ -37,6 +43,37 @@ def get_asset_path( ]) +def pluck(obj, *props, **additions): + return dict({k: v for k, v in obj.items() if k in props}, **additions) + + +def first_key(data, *keys): + for key in keys: + value = data.get(key) + if value: + return key, value + return None, None + + +@functools.lru_cache() +def integrity_hash_from_file(filename): + with open(filename, 'rb') as f: + h = hashlib.sha384(f.read()) + + return 'sha384-{}'.format( + base64.b64encode(h.digest()).decode() + ) + + +@functools.lru_cache() +def integrity_hash_from_package(namespace, path): + h = hashlib.sha384(pkgutil.get_data(namespace, path)) + + return 'sha384-{}'.format( + base64.b64encode(h.digest()).decode() + ) + + class AttributeDict(dict): """ Dictionary subclass enabling attribute lookup/assignment of keys/values. From fe0b6bb03e52f9330ab99fc36399724a3c2f5a33 Mon Sep 17 00:00:00 2001 From: Philippe Duval Date: Tue, 30 Oct 2018 17:05:20 -0400 Subject: [PATCH 02/10] Add integrity hash to generated script/link tags. --- dash/dash.py | 56 ++++++++++++++++++++++++++++++++++++++--------- dash/resources.py | 36 +++++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 11 deletions(-) diff --git a/dash/dash.py b/dash/dash.py index 0a68d70bce..3368d89a18 100644 --- a/dash/dash.py +++ b/dash/dash.py @@ -25,6 +25,7 @@ from ._utils import interpolate_str as _interpolate from ._utils import format_tag as _format_tag from ._utils import get_asset_path as _get_asset_path +from ._utils import pluck as _pluck from . import _configs @@ -313,6 +314,7 @@ def _collect_and_register_resources(self, resources): def _relative_url_path(relative_package_path='', namespace=''): # track the registered packages + # FIXME it append a new path each time the index is served ? if namespace in self.registered_paths: self.registered_paths[namespace].append(relative_package_path) else: @@ -332,23 +334,34 @@ def _relative_url_path(relative_package_path='', namespace=''): modified ) + def pack(src, **kwargs): + return dict(kwargs, src=src) + srcs = [] for resource in resources: if 'relative_package_path' in resource: if isinstance(resource['relative_package_path'], str): - srcs.append(_relative_url_path(**resource)) + srcs.append( + pack( + _relative_url_path( + resource['relative_package_path'], + resource['namespace']), + **resource) + ) else: for rel_path in resource['relative_package_path']: - srcs.append(_relative_url_path( + srcs.append(pack(_relative_url_path( relative_package_path=rel_path, - namespace=resource['namespace'] - )) + namespace=resource['namespace'], + ), **resource)) elif 'external_url' in resource: if isinstance(resource['external_url'], str): - srcs.append(resource['external_url']) + srcs.append( + pack(resource['external_url'], **resource) + ) else: for url in resource['external_url']: - srcs.append(url) + srcs.append(pack(url)) elif 'absolute_path' in resource: raise Exception( 'Serving files from absolute_path isn\'t supported yet' @@ -357,7 +370,7 @@ def _relative_url_path(relative_package_path='', namespace=''): static_url = self.get_asset_url(resource['asset_path']) # Add a bust query param static_url += '?m={}'.format(resource['ts']) - srcs.append(static_url) + srcs.append(pack(static_url, **resource)) return srcs def _generate_css_dist_html(self): @@ -365,7 +378,22 @@ def _generate_css_dist_html(self): self._collect_and_register_resources(self.css.get_all_css()) return '\n'.join([ - _format_tag('link', link, opened=True) + _format_tag( + 'link', + _pluck( + link, + 'integrity', + 'crossorigin', + 'media', + 'charset', + 'rev', + 'target', + 'type', + href=link.get('src'), + rel='stylesheet' + ), + opened=True + ) if isinstance(link, dict) else ''.format(link) for link in links @@ -392,7 +420,15 @@ def _generate_scripts_html(self): )) return '\n'.join([ - _format_tag('script', src) + _format_tag('script', _pluck( + src, + 'src', + 'integrity', + 'crossorigin', + 'charset', + 'async', + 'defer', + 'type')) if isinstance(src, dict) else ''.format(src) for src in srcs @@ -492,7 +528,7 @@ def index(self, *args, **kwargs): # pylint: disable=unused-argument checks = ( (_re_index_entry_id.search(index), '#react-entry-point'), (_re_index_config_id.search(index), '#_dash-configs'), - (_re_index_scripts_id.search(index), 'dash-renderer'), + # (_re_index_scripts_id.search(index), 'dash-renderer'), ) missing = [missing for check, missing in checks if not check] diff --git a/dash/resources.py b/dash/resources.py index aa1ce871d9..eab3b2795c 100644 --- a/dash/resources.py +++ b/dash/resources.py @@ -4,6 +4,7 @@ import os from .development.base_component import Component +from ._utils import first_key, integrity_hash_from_file, integrity_hash_from_package # pylint: disable=old-style-class @@ -20,6 +21,7 @@ def _filter_resources(self, all_resources, dev_bundles=False): filtered_resources = [] for s in all_resources: filtered_resource = {} + added = False if 'namespace' in s: filtered_resource['namespace'] = s['namespace'] if 'external_url' in s and not self.config.serve_locally: @@ -36,6 +38,9 @@ def _filter_resources(self, all_resources, dev_bundles=False): filtered_resource['absolute_path'] = s['absolute_path'] elif 'asset_path' in s: info = os.stat(s['filepath']) + filtered_resource['integrity'] = integrity_hash_from_file( + s['filepath']) + filtered_resource['crossorigin'] = 'anonymous' filtered_resource['asset_path'] = s['asset_path'] filtered_resource['ts'] = info.st_mtime elif self.config.serve_locally: @@ -54,7 +59,36 @@ def _filter_resources(self, all_resources, dev_bundles=False): ) ) - filtered_resources.append(filtered_resource) + if 'integrity' not in filtered_resource \ + and 'namespace' in filtered_resource: + # Get the resource key and value. + key, filename = first_key( + filtered_resource, + 'external_url', + 'dev_package_path', + 'relative_package_path' + ) + if isinstance(filename, list): + # flatten these dependencies + for f in filename: + filtered_resources.append({ + key: f, + 'integrity': integrity_hash_from_package( + filtered_resource['namespace'], f.split('/')[-1]), + 'crossorigin': 'anonymous', + 'namespace': filtered_resource['namespace'] + }) + added = True + else: + filename = filename.split('/')[-1] + filtered_resource['integrity'] = integrity_hash_from_package( + filtered_resource['namespace'], + filename + ) + filtered_resource['crossorigin'] = 'anonymous' + + if not added: + filtered_resources.append(filtered_resource) return filtered_resources From e8146f1acb50227e2aaca8ee87ee4ddd29cd5fae Mon Sep 17 00:00:00 2001 From: Philippe Duval Date: Tue, 30 Oct 2018 17:09:28 -0400 Subject: [PATCH 03/10] pylint fix. --- dash/resources.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dash/resources.py b/dash/resources.py index eab3b2795c..a39c38a151 100644 --- a/dash/resources.py +++ b/dash/resources.py @@ -17,6 +17,7 @@ def __init__(self, resource_name, layout): def append_resource(self, resource): self._resources.append(resource) + # pylint: disable=too-many-branches def _filter_resources(self, all_resources, dev_bundles=False): filtered_resources = [] for s in all_resources: From ef94ab49fe997579a9b14638e959ac11d527929f Mon Sep 17 00:00:00 2001 From: Philippe Duval Date: Tue, 30 Oct 2018 17:19:07 -0400 Subject: [PATCH 04/10] Use a custom memoize instead of lru_cache for py2 support. --- dash/_utils.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/dash/_utils.py b/dash/_utils.py index 099995e4c9..c65ac73568 100644 --- a/dash/_utils.py +++ b/dash/_utils.py @@ -4,6 +4,21 @@ import pkgutil +def memoize(func): + results = {} + + @functools.wraps(func) + def wrapper(*args, **kwargs): + key = hash((args, frozenset(kwargs))) + cached = results.get(key) + if cached: + return cached + results[key] = r = func(*args, **kwargs) + return r + + return wrapper + + def interpolate_str(template, **data): s = template for k, v in data.items(): @@ -55,7 +70,7 @@ def first_key(data, *keys): return None, None -@functools.lru_cache() +@memoize def integrity_hash_from_file(filename): with open(filename, 'rb') as f: h = hashlib.sha384(f.read()) @@ -65,7 +80,7 @@ def integrity_hash_from_file(filename): ) -@functools.lru_cache() +@memoize def integrity_hash_from_package(namespace, path): h = hashlib.sha384(pkgutil.get_data(namespace, path)) From 027d709e61511a1cc8aa54ef4431259688e2383f Mon Sep 17 00:00:00 2001 From: Philippe Duval Date: Tue, 30 Oct 2018 17:23:35 -0400 Subject: [PATCH 05/10] Fix line length & disable old-style-class globally. --- .pylintrc | 3 ++- dash/resources.py | 15 +++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.pylintrc b/.pylintrc index fba35186cc..fcf0968f10 100644 --- a/.pylintrc +++ b/.pylintrc @@ -57,7 +57,8 @@ confidence= disable=fixme, missing-docstring, invalid-name, - too-many-lines + too-many-lines, + old-style-class # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where diff --git a/dash/resources.py b/dash/resources.py index a39c38a151..552934826a 100644 --- a/dash/resources.py +++ b/dash/resources.py @@ -4,7 +4,8 @@ import os from .development.base_component import Component -from ._utils import first_key, integrity_hash_from_file, integrity_hash_from_package +from ._utils import \ + first_key, integrity_hash_from_file, integrity_hash_from_package # pylint: disable=old-style-class @@ -75,17 +76,19 @@ def _filter_resources(self, all_resources, dev_bundles=False): filtered_resources.append({ key: f, 'integrity': integrity_hash_from_package( - filtered_resource['namespace'], f.split('/')[-1]), + filtered_resource['namespace'], + f.split('/')[-1]), 'crossorigin': 'anonymous', 'namespace': filtered_resource['namespace'] }) added = True else: filename = filename.split('/')[-1] - filtered_resource['integrity'] = integrity_hash_from_package( - filtered_resource['namespace'], - filename - ) + filtered_resource['integrity'] = \ + integrity_hash_from_package( + filtered_resource['namespace'], + filename + ) filtered_resource['crossorigin'] = 'anonymous' if not added: From aec70fb94bceb0e781ae2cfecc95cbe6018defb0 Mon Sep 17 00:00:00 2001 From: Philippe Duval Date: Wed, 31 Oct 2018 12:37:16 -0400 Subject: [PATCH 06/10] :camel: integrity hash generation. --- dash/_utils.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dash/_utils.py b/dash/_utils.py index c65ac73568..e90459bf8e 100644 --- a/dash/_utils.py +++ b/dash/_utils.py @@ -70,10 +70,8 @@ def first_key(data, *keys): return None, None -@memoize -def integrity_hash_from_file(filename): - with open(filename, 'rb') as f: - h = hashlib.sha384(f.read()) +def _integrity_hash(bytes): + h = hashlib.sha384(bytes) return 'sha384-{}'.format( base64.b64encode(h.digest()).decode() @@ -81,12 +79,14 @@ def integrity_hash_from_file(filename): @memoize -def integrity_hash_from_package(namespace, path): - h = hashlib.sha384(pkgutil.get_data(namespace, path)) +def integrity_hash_from_file(filename): + with open(filename, 'rb') as f: + return _integrity_hash(f.read()) - return 'sha384-{}'.format( - base64.b64encode(h.digest()).decode() - ) + +@memoize +def integrity_hash_from_package(namespace, path): + return _integrity_hash(pkgutil.get_data(namespace, path)) class AttributeDict(dict): From e07ed47716081edb9535af6acc0b88cd736dc82a Mon Sep 17 00:00:00 2001 From: Philippe Duval Date: Wed, 31 Oct 2018 12:39:49 -0400 Subject: [PATCH 07/10] Find local file for unpkg url. --- dash/resources.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/dash/resources.py b/dash/resources.py index 552934826a..7ffae70b1c 100644 --- a/dash/resources.py +++ b/dash/resources.py @@ -8,6 +8,25 @@ first_key, integrity_hash_from_file, integrity_hash_from_package +def find_unpkg(value, relative_package_paths): + # find the local file for a unpkg url. + # The structure of the _js_dist/_css_dist does not allow + # for easy translation between local and external dependencies. + v = value.replace('https://unpkg.com/', '') + s = v.split('/') + lib, version = s[0].split('@') + filename = s[-1] + ext = filename.split('.')[-1] + + for i in relative_package_paths: + if (i == filename + and i not in [ + 'index.js', 'index.css', 'style.css', 'style.min.css', + 'styles.css', 'styles.min.css']) \ + or (lib in i and version in i and i.endswith(ext)): + return i + + # pylint: disable=old-style-class class Resources: def __init__(self, resource_name, layout): @@ -28,6 +47,7 @@ def _filter_resources(self, all_resources, dev_bundles=False): filtered_resource['namespace'] = s['namespace'] if 'external_url' in s and not self.config.serve_locally: filtered_resource['external_url'] = s['external_url'] + filtered_resource['local_file'] = s['relative_package_path'] elif 'dev_package_path' in s and dev_bundles: filtered_resource['relative_package_path'] = ( s['dev_package_path'] @@ -63,7 +83,6 @@ def _filter_resources(self, all_resources, dev_bundles=False): if 'integrity' not in filtered_resource \ and 'namespace' in filtered_resource: - # Get the resource key and value. key, filename = first_key( filtered_resource, 'external_url', @@ -71,19 +90,23 @@ def _filter_resources(self, all_resources, dev_bundles=False): 'relative_package_path' ) if isinstance(filename, list): - # flatten these dependencies + # flatten these dependencies and add a hash for each. for f in filename: + local_file = filtered_resource.get('local_file') + filename = find_unpkg(f, local_file) \ + if local_file else f.split('/')[-1] filtered_resources.append({ key: f, 'integrity': integrity_hash_from_package( filtered_resource['namespace'], - f.split('/')[-1]), + filename), 'crossorigin': 'anonymous', 'namespace': filtered_resource['namespace'] }) added = True else: - filename = filename.split('/')[-1] + filename = filtered_resource.get( + 'local_file', filename.split('/')[-1]) filtered_resource['integrity'] = \ integrity_hash_from_package( filtered_resource['namespace'], From 222533b6d82abe37fce1220e3210386b36676072 Mon Sep 17 00:00:00 2001 From: Philippe Duval Date: Wed, 31 Oct 2018 12:48:42 -0400 Subject: [PATCH 08/10] pylint fixes --- dash/_utils.py | 4 ++-- dash/resources.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dash/_utils.py b/dash/_utils.py index e90459bf8e..8940a07972 100644 --- a/dash/_utils.py +++ b/dash/_utils.py @@ -70,8 +70,8 @@ def first_key(data, *keys): return None, None -def _integrity_hash(bytes): - h = hashlib.sha384(bytes) +def _integrity_hash(b): + h = hashlib.sha384(b) return 'sha384-{}'.format( base64.b64encode(h.digest()).decode() diff --git a/dash/resources.py b/dash/resources.py index 7ffae70b1c..d9a060ec0f 100644 --- a/dash/resources.py +++ b/dash/resources.py @@ -8,6 +8,7 @@ first_key, integrity_hash_from_file, integrity_hash_from_package +# pylint: disable=inconsistent-return-statements def find_unpkg(value, relative_package_paths): # find the local file for a unpkg url. # The structure of the _js_dist/_css_dist does not allow @@ -19,8 +20,7 @@ def find_unpkg(value, relative_package_paths): ext = filename.split('.')[-1] for i in relative_package_paths: - if (i == filename - and i not in [ + if (i == filename and i not in [ 'index.js', 'index.css', 'style.css', 'style.min.css', 'styles.css', 'styles.min.css']) \ or (lib in i and version in i and i.endswith(ext)): From ee9c1d7c49978b7c4f63ddd0bcacbf6a74a30b2b Mon Sep 17 00:00:00 2001 From: Philippe Duval Date: Wed, 31 Oct 2018 13:10:29 -0400 Subject: [PATCH 09/10] Fix link that already got an href value. --- dash/dash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dash/dash.py b/dash/dash.py index 3368d89a18..41cd7a1c79 100644 --- a/dash/dash.py +++ b/dash/dash.py @@ -389,7 +389,7 @@ def _generate_css_dist_html(self): 'rev', 'target', 'type', - href=link.get('src'), + href=link.get('src', link.get('href')), rel='stylesheet' ), opened=True From d45b209704d0b8a27e2213ce5b6e3e9eb33057be Mon Sep 17 00:00:00 2001 From: Philippe Duval Date: Mon, 5 Nov 2018 10:19:01 -0500 Subject: [PATCH 10/10] Fix memoize kwargs. --- dash/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dash/_utils.py b/dash/_utils.py index 8940a07972..d91eab8a86 100644 --- a/dash/_utils.py +++ b/dash/_utils.py @@ -9,7 +9,7 @@ def memoize(func): @functools.wraps(func) def wrapper(*args, **kwargs): - key = hash((args, frozenset(kwargs))) + key = hash((args, frozenset(kwargs.items()))) cached = results.get(key) if cached: return cached