diff --git a/pyswagger/core.py b/pyswagger/core.py index 91b563f..15cbbe5 100644 --- a/pyswagger/core.py +++ b/pyswagger/core.py @@ -1,19 +1,19 @@ from __future__ import absolute_import from .getter import UrlGetter, LocalGetter +from .resolve import SwaggerResolver from .primitives import SwaggerPrimitive from .spec.v1_2.parser import ResourceListContext from .spec.v2_0.parser import SwaggerContext from .spec.v2_0.objects import Operation +from .spec.base import BaseObj from .scan import Scanner from .scanner import TypeReduce, CycleDetector from .scanner.v1_2 import Upgrade -from .scanner.v2_0 import AssignParent, Resolve, PatchObject, YamlFixer, Aggregate +from .scanner.v2_0 import AssignParent, Merge, Resolve, PatchObject, YamlFixer, Aggregate, NormalizeRef from pyswagger import utils, errs, consts -import inspect import base64 import six import weakref -import os import logging @@ -33,11 +33,10 @@ class SwaggerApp(object): sc_path: ('/', '#/paths') } - def __init__(self, url=None, app_cache=None, url_load_hook=None, sep=consts.private.SCOPE_SEPARATOR, prim=None): + def __init__(self, url=None, url_load_hook=None, sep=consts.private.SCOPE_SEPARATOR, prim=None): """ constructor :param url str: url of swagger.json - :param dict app_cache: a url map shared by SwaggerApp(s), mapping from url to SwaggerApp :param func url_load_hook: a way to redirect url to a accessible place. for self testing. :param sep str: separator used by pyswager.utils.ScopeDict :param prim pyswagger.primitives.SwaggerPrimitive: factory for primitives in Swagger. @@ -54,15 +53,14 @@ def __init__(self, url=None, app_cache=None, url_load_hook=None, sep=consts.priv self.__schemes = [] self.__url=url - # a map from url to SwaggerApp - self.__app_cache = {} if app_cache == None else app_cache + # a map from json-reference to + # - spec.BaseObj + # - a map from json-pointer to spec.BaseObj + self.__objs = {} + self.__resolver = SwaggerResolver(url_load_hook) # keep a string reference to SwaggerApp when resolve self.__strong_refs = [] - # things to make unittest easier, - # all urls to load json would go through this hook - self.__url_load_hook = url_load_hook - # allow init App-wised SCOPE_SEPARATOR self.__sep = sep @@ -139,12 +137,6 @@ def url(self): """ return self.__url - @property - def _app_cache(self): - """ internal usage - """ - return self.__app_cache - @property def prim_factory(self): """ primitive factory used by this app @@ -153,41 +145,18 @@ def prim_factory(self): """ return self.__prim - def _load_obj(self, url, getter=None, parser=None): - """ + def load_obj(self, jref, getter=None, parser=None): + """ load a object(those in spec._version_.objects) from a JSON reference. """ - if url in self.__app_cache: - logger.info('{0} hit cache'.format(url)) - - # look into cache first - return - - # apply hook when use this url to load - # note that we didn't cache SwaggerApp with this local_url - local_url = url if not self.__url_load_hook else self.__url_load_hook(url) - - logger.info('{0} patch to {1}'.format(url, local_url)) - - if not getter: - getter = UrlGetter - p = six.moves.urllib.parse.urlparse(local_url) - if p.scheme == 'file' and p.path: - getter = LocalGetter(os.path.join(p.netloc, p.path)) - - if inspect.isclass(getter): - # default initialization is passing the url - # you can override this behavior by passing an - # initialized getter object. - getter = getter(local_url) + obj = self.__resolver.resolve(jref, getter) # get root document to check its swagger version. - obj, _ = six.advance_iterator(getter) tmp = {'_tmp_': {}} version = utils.get_swagger_version(obj) if version == '1.2': # swagger 1.2 with ResourceListContext(tmp, '_tmp_') as ctx: - ctx.parse(getter, obj) + ctx.parse(obj, jref, self.__resolver, getter) elif version == '2.0': # swagger 2.0 with SwaggerContext(tmp, '_tmp_') as ctx: @@ -198,13 +167,55 @@ def _load_obj(self, url, getter=None, parser=None): version = tmp['_tmp_'].__swagger_version__ if hasattr(tmp['_tmp_'], '__swagger_version__') else version else: - raise NotImplementedError('Unsupported Swagger Version: {0} from {1}'.format(version, url)) + raise NotImplementedError('Unsupported Swagger Version: {0} from {1}'.format(version, jref)) + + if not tmp['_tmp_']: + raise Exception('Unable to parse object from {0}'.format(jref)) logger.info('version: {0}'.format(version)) - self.__app_cache[url] = weakref.proxy(self) # avoid circular reference - self.__version = version - self.__raw = tmp['_tmp_'] + return tmp['_tmp_'], version + + def prepare_obj(self, obj, jref): + """ basic preparation of an object(those in sepc._version_.objects), + and cache the 'prepared' object. + """ + if not obj: + raise Exception('unexpected, passing {0}:{1} to prepare'.format(obj, jref)) + + s = Scanner(self) + if self.version == '1.2': + # upgrade from 1.2 to 2.0 + converter = Upgrade(self.__sep) + s.scan(root=obj, route=[converter]) + obj = converter.swagger + + if not obj: + raise Exception('unable to upgrade from 1.2: {0}'.format(jref)) + + s.scan(root=obj, route=[AssignParent()]) + + # normalize $ref + url, jp = utils.jr_split(jref) + s.scan(root=obj, route=[NormalizeRef(url)]) + # fix for yaml that treat response code as number + s.scan(root=obj, route=[YamlFixer()], leaves=[Operation]) + + # cache this object + if url not in self.__objs: + if jp == '#': + self.__objs[url] = obj + else: + self.__objs[url] = {jp: obj} + else: + if not isinstance(self.__objs[url], dict): + raise Exception('it should be able to resolve with BaseObj') + self.__objs[url].update({jp: obj}) + + # pre resolve Schema Object + # note: make sure this object is cached before using 'Resolve' scanner + s.scan(root=obj, route=[Resolve()]) + return obj def _validate(self): """ check if this Swagger API valid or not. @@ -232,44 +243,8 @@ def _validate(self): s.scan(route=[v], root=self.__raw) return v.errs - def _prepare_obj(self, strict=True): - """ - """ - if self.__root: - return - - s = Scanner(self) - self.validate(strict=strict) - - if self.version == '1.2': - converter = Upgrade(self.__sep) - s.scan(root=self.raw, route=[converter]) - obj = converter.swagger - - # We only have to run this scanner when upgrading from 1.2. - # Mainly because we initial BaseObj via NullContext - s.scan(root=obj, route=[AssignParent()]) - - self.__root = obj - elif self.version == '2.0': - s.scan(root=self.raw, route=[YamlFixer()], leaves=[Operation]) - self.__root = self.raw - else: - raise NotImplementedError('Unsupported Version: {0}'.format(self.__version)) - - if hasattr(self.__root, 'schemes') and self.__root.schemes: - if len(self.__root.schemes) > 0: - self.__schemes = self.__root.schemes - else: - # extract schemes from the url to load spec - self.__schemes = [six.moves.urlparse(self.__url).schemes] - - s.scan(root=self.__root, route=[Resolve()]) - s.scan(root=self.__root, route=[PatchObject()]) - s.scan(root=self.__root, route=[Aggregate()]) - @classmethod - def load(kls, url, getter=None, parser=None, app_cache=None, url_load_hook=None, sep=consts.private.SCOPE_SEPARATOR, prim=None): + def load(kls, url, getter=None, parser=None, url_load_hook=None, sep=consts.private.SCOPE_SEPARATOR, prim=None): """ load json as a raw SwaggerApp :param str url: url of path of Swagger API definition @@ -290,9 +265,10 @@ def load(kls, url, getter=None, parser=None, app_cache=None, url_load_hook=None, logger.info('load with [{0}]'.format(url)) url = utils.normalize_url(url) - app = kls(url, app_cache=app_cache, url_load_hook=url_load_hook, sep=sep, prim=prim) - - app._load_obj(url, getter, parser) + app = kls(url, url_load_hook=url_load_hook, sep=sep, prim=prim) + app.__raw, app.__version = app.load_obj(url, getter=getter, parser=parser) + if app.__version not in ['1.2', '2.0']: + raise NotImplementedError('Unsupported Version: {0}'.format(self.__version)) # update schem if any p = six.moves.urllib.parse.urlparse(url) @@ -321,10 +297,22 @@ def prepare(self, strict=True): :param bool strict: when in strict mode, exception would be raised if not valid. """ - self._prepare_obj(strict=strict) + self.validate(strict=strict) + self.__root = self.prepare_obj(self.raw, self.__url) + + if hasattr(self.__root, 'schemes') and self.__root.schemes: + if len(self.__root.schemes) > 0: + self.__schemes = self.__root.schemes + else: + # extract schemes from the url to load spec + self.__schemes = [six.moves.urlparse(self.__url).schemes] - # reducer for Operation s = Scanner(self) + s.scan(root=self.__root, route=[Merge()]) + s.scan(root=self.__root, route=[PatchObject()]) + s.scan(root=self.__root, route=[Aggregate()]) + + # reducer for Operation tr = TypeReduce(self.__sep) cy = CycleDetector() s.scan(root=self.__root, route=[tr, cy]) @@ -381,28 +369,40 @@ def resolve(self, jref, parser=None): if jref == None or len(jref) == 0: raise ValueError('Empty Path is not allowed') + obj = None url, jp = utils.jr_split(jref) if url: - if url not in self.__app_cache: - # This loaded SwaggerApp would be kept in app_cache. - app = SwaggerApp.load(url, parser=parser, app_cache=self.__app_cache, url_load_hook=self.__url_load_hook) - app.prepare() - - # nothing but only keeping a strong reference of - # loaded SwaggerApp. - self.__strong_refs.append(app) - - return self.__app_cache[url].resolve(jp) + # check cacahed object against json reference by + # comparing url first, and find those object prefixed with + # the JSON pointer. + o = self.__objs.get(url, None) + if o: + if isinstance(o, BaseObj): + obj = o.resolve(utils.jp_split(jp)[1:]) + elif isinstance(o, dict): + for k, v in six.iteritems(o): + if jp.startswith(k): + obj = v.resolve(utils.jp_split(jp[len(k):])[1:]) + break + else: + raise Exception('Unknown Cached Object: {0}'.format(str(type(o)))) - if not jp.startswith('#'): - raise ValueError('Invalid Path, root element should be \'#\', but [{0}]'.format(jref)) + # this object is not loaded yet, load it + if obj == None: + obj, _ = self.load_obj(jref, parser=parser) + if obj: + obj = self.prepare_obj(obj, jref) + else: + # a local reference, 'jref' is just a json-pointer + if not jp.startswith('#'): + raise ValueError('Invalid Path, root element should be \'#\', but [{0}]'.format(jref)) - obj = self.root.resolve(utils.jp_split(jp)[1:]) # heading element is #, mapping to self.root + obj = self.root.resolve(utils.jp_split(jp)[1:]) # heading element is #, mapping to self.root if obj == None: raise ValueError('Unable to resolve path, [{0}]'.format(jref)) - if isinstance(obj, (six.string_types, int, list, dict)): + if isinstance(obj, (six.string_types, six.integer_types, list, dict)): return obj return weakref.proxy(obj) diff --git a/pyswagger/getter.py b/pyswagger/getter.py index f706af8..4604190 100644 --- a/pyswagger/getter.py +++ b/pyswagger/getter.py @@ -27,8 +27,7 @@ def __next__(self): if len(self.urls) == 0: raise StopIteration - path, name = self.urls.pop(0) - obj = self.load(path) + obj = self.load(self.urls.pop(0)) # make sure data is string type if isinstance(obj, six.binary_type): @@ -45,15 +44,7 @@ def __next__(self): except ValueError: raise Exception('Unknown format startswith {0} ...'.format(obj[:10])) - # find urls to retrieve from resource listing file - if name == '': - urls = self.__find_urls(obj) - self.urls.extend(zip( - map(lambda u: self.base_path + u, urls), - map(lambda u: u[1:], urls) - )) - - return obj, name + return obj def load(self, path): """ load the resource, and return for parsing. @@ -63,25 +54,6 @@ def load(self, path): """ raise NotImplementedError() - def __find_urls(self, obj): - """ helper function to located relative url in Resource Listing object. - - :param dict obj: json of Resource Listing object. - :return: urls of resources - :rtype: a list of str - """ - urls = [] - if private.SCHEMA_APIS in obj: - # This is a Swagger 1.2 spec, need to load subsequent resource files. - if isinstance(obj[private.SCHEMA_APIS], list): - for api in obj[private.SCHEMA_APIS]: - urls.append(api[private.SCHEMA_PATH]) - else: - raise TypeError('Invalid type of apis: ' + type(obj[private.SCHEMA_APIS])) - - return urls - - class LocalGetter(Getter): """ default getter implmenetation for local resource file """ @@ -91,12 +63,12 @@ def __init__(self, path): for n in private.SWAGGER_FILE_NAMES: if self.base_path.endswith(n): self.base_path = os.path.dirname(self.base_path) - self.urls = [(path, '')] + self.urls = [path] break else: p = os.path.join(path, n) if os.path.isfile(p): - self.urls = [(p, '')] + self.urls = [p] break else: # there is no file matched predefined file name: @@ -109,10 +81,15 @@ def __init__(self, path): for e in [private.FILE_EXT_JSON, private.FILE_EXT_YAML]: if ext.endswith(e): self.base_path = os.path.dirname(path) - self.urls=[(path, '')] + self.urls = [path] break else: - raise ValueError('Unable to locate resource file: [{0}]'.format(path)) + for e in [private.FILE_EXT_JSON, private.FILE_EXT_YAML]: + if os.path.isfile(path + '.' + e): + self.urls = [path + '.' + e] + break + else: + raise ValueError('Unable to locate resource file: [{0}]'.format(path)) def load(self, path): ret = None @@ -147,7 +124,7 @@ def __init__(self, path): super(UrlGetter, self).__init__(path) if self.base_path.endswith('/'): self.base_path = self.base_path[:-1] - self.urls = [(path, '')] + self.urls = [path] def load(self, path): diff --git a/pyswagger/io.py b/pyswagger/io.py index ed419ce..03b9a37 100644 --- a/pyswagger/io.py +++ b/pyswagger/io.py @@ -1,6 +1,6 @@ from __future__ import absolute_import from .primitives.comm import PrimJSONEncoder -from .utils import deref +from .utils import final from pyswagger import errs from uuid import uuid4 import six @@ -313,8 +313,8 @@ def apply_with(self, status=None, raw=None, header=None): if status != None: self.__status = status - r = (deref(self.__op.responses.get(str(self.__status), None)) or - deref(self.__op.responses.get('default', None))) + r = (final(self.__op.responses.get(str(self.__status), None)) or + final(self.__op.responses.get('default', None))) if raw != None: # update 'raw' diff --git a/pyswagger/primitives/render.py b/pyswagger/primitives/render.py index c1a8943..3cebe12 100644 --- a/pyswagger/primitives/render.py +++ b/pyswagger/primitives/render.py @@ -1,6 +1,6 @@ from __future__ import absolute_import from ..spec.v2_0.objects import Parameter, Operation, Schema -from ..utils import deref, from_iso8601 +from ..utils import deref, final, from_iso8601 from decimal import Decimal import random import six @@ -167,8 +167,7 @@ def _get(self, _type, _format=None): return None if r == None else r.get(_format, None) def _generate(self, obj, opt): - obj = deref(obj) - obj = obj.final if isinstance(obj, Schema) else obj + obj = final(deref(obj)) type_ = getattr(obj, 'type', None) template = opt['object_template'] out = None diff --git a/pyswagger/resolve.py b/pyswagger/resolve.py new file mode 100644 index 0000000..9111be6 --- /dev/null +++ b/pyswagger/resolve.py @@ -0,0 +1,72 @@ +from __future__ import absolute_import +from .utils import jr_split, jp_split +from .getter import UrlGetter, LocalGetter +import six +import os +import inspect +import logging + + +logger = logging.getLogger(__name__) + + +class SwaggerResolver(object): + """ JSON Reference Resolver: + resolving a JSON reference to a raw object (dict), + then return and cache it. + """ + + def __init__(self, url_load_hook): + """ + """ + # a map from url to loaded json/yaml + self.__cache = {} + + # things to make unittest easier, + # all urls to load json would go through this hook + self.__url_load_hook = url_load_hook + + def resolve(self, jref, getter=None): + """ + """ + url, jp = jr_split(jref) + + # apply hook when use this url to load + # note that we didn't cache SwaggerApp with this local_url + local_url = self.__url_load_hook(url) if self.__url_load_hook else url + + logger.info('{0} patch to {1}'.format(url, local_url)) + + # check cache + obj = self.__cache.get(url, None) + if not obj: + # load that object + if not getter: + getter = UrlGetter + p = six.moves.urllib.parse.urlparse(local_url) + if p.scheme == 'file' and p.path: + getter = LocalGetter(os.path.join(p.netloc, p.path)) + + if inspect.isclass(getter): + # default initialization is passing the url + # you can override this behavior by passing an + # initialized getter object. + getter = getter(local_url) + + obj = six.advance_iterator(getter) + self.__cache[url] = obj if obj else None + + if obj: + ts = jp_split(jp)[1:] + while len(ts) > 0: + t = ts.pop(0) + if isinstance(obj, list): + obj = obj[int(t)] + elif isinstance(obj, dict): + obj = obj[t] + else: + raise Exception('Invalid type to resolve json-pointer: {0}'.format(str(type(obj)))) + else: + raise Exception('Unable to resolve: {0}'.format(jref)) + + return obj diff --git a/pyswagger/scanner/cycle_detector.py b/pyswagger/scanner/cycle_detector.py index 166e2cc..893a687 100644 --- a/pyswagger/scanner/cycle_detector.py +++ b/pyswagger/scanner/cycle_detector.py @@ -1,5 +1,5 @@ from __future__ import absolute_import -from ..utils import normalize_jr, walk +from ..utils import walk from ..scan import Dispatcher from ..spec.v2_0.objects import ( Schema, @@ -7,12 +7,18 @@ Response, PathItem, ) +from ..spec.v2_0.parser import ( + SchemaContext, + ParameterContext, + ResponseContext, + PathItemContext, + ) import functools import six -def _out(app, path): - obj = app.resolve(normalize_jr(path, app.url)) - r = getattr(obj, 'norm_ref') +def _out(app, parser, path): + obj = app.resolve(path, parser=parser) + r = getattr(obj, '$ref') return [r] if r else [] def _schema_out_obj(obj, out=None): @@ -30,14 +36,14 @@ def _schema_out_obj(obj, out=None): if obj.items: out = _schema_out_obj(obj.items, out) - r = getattr(obj, 'norm_ref') + r = getattr(obj, '$ref') if r: out.append(r) return out def _schema_out(app, path): - obj = app.resolve(normalize_jr(path, app.url)) + obj = app.resolve(path, parser=SchemaContext) return [] if obj == None else _schema_out_obj(obj) @@ -66,7 +72,7 @@ def _schema(self, path, _, app): def _parameter(self, path, _, app): self.cycles['parameter'] = walk( path, - functools.partial(_out, app), + functools.partial(_out, app, ParameterContext), self.cycles['parameter'] ) @@ -74,7 +80,7 @@ def _parameter(self, path, _, app): def _response(self, path, _, app): self.cycles['response'] = walk( path, - functools.partial(_out, app), + functools.partial(_out, app, ResponseContext), self.cycles['response'] ) @@ -82,7 +88,7 @@ def _response(self, path, _, app): def _path_item(self, path, _, app): self.cycles['path_item'] = walk( path, - functools.partial(_out, app), + functools.partial(_out, app, PathItemContext), self.cycles['path_item'] ) diff --git a/pyswagger/scanner/v2_0/__init__.py b/pyswagger/scanner/v2_0/__init__.py index 2edee42..c68cd03 100644 --- a/pyswagger/scanner/v2_0/__init__.py +++ b/pyswagger/scanner/v2_0/__init__.py @@ -3,3 +3,5 @@ from .patch_obj import PatchObject from .yaml import YamlFixer from .aggt import Aggregate +from .norm_ref import NormalizeRef +from .merge import Merge diff --git a/pyswagger/scanner/v2_0/merge.py b/pyswagger/scanner/v2_0/merge.py new file mode 100644 index 0000000..107153f --- /dev/null +++ b/pyswagger/scanner/v2_0/merge.py @@ -0,0 +1,60 @@ +from __future__ import absolute_import +from ...errs import CycleDetectionError +from ...scan import Dispatcher +from ...spec.v2_0.parser import ( + ParameterContext, + ResponseContext, + PathItemContext + ) +from ...spec.v2_0.objects import ( + Parameter, + Response, + PathItem, + ) +from ...spec.base import NullContext +from ...utils import CycleGuard + + +def _merge(obj, app, creator, parser): + """ resolve $ref, and inject/merge referenced object to self. + This operation should be carried in a cascade manner. + """ + result = creator(NullContext()) + result.merge(obj, parser) + + guard = CycleGuard() + guard.update(obj) + + r = getattr(obj, '$ref') + while r and len(r) > 0: + ro = app.resolve(r, parser) + if ro.__class__ != obj.__class__: + raise TypeError('Referenced Type mismatch: {0}'.format(r)) + try: + guard.update(ro) + except CycleDetectionError: + # avoid infinite loop, + # cycle detection has a dedicated scanner. + break + + result.merge(ro, parser) + r = getattr(ro, '$ref') + return result + +class Merge(object): + """ pre-merge these objects with '$ref' """ + + class Disp(Dispatcher): pass + + @Disp.register([Parameter]) + def _parameter(self, _, obj, app): + obj.update_field('final', _merge(obj, app, Parameter, ParameterContext)) + + @Disp.register([Response]) + def _response(self, _, obj, app): + obj.update_field('final', _merge(obj, app, Response, ResponseContext)) + + @Disp.register([PathItem]) + def _path_item(self, _, obj, app): + obj.merge(_merge(obj, app, PathItem, PathItemContext), PathItemContext) + diff --git a/pyswagger/scanner/v2_0/norm_ref.py b/pyswagger/scanner/v2_0/norm_ref.py new file mode 100644 index 0000000..758036b --- /dev/null +++ b/pyswagger/scanner/v2_0/norm_ref.py @@ -0,0 +1,22 @@ +from __future__ import absolute_import +from ...scan import Dispatcher +from ...utils import normalize_jr +from ...spec.v2_0.objects import ( + Schema, + Parameter, + Response, + PathItem, + ) + + +class NormalizeRef(object): + """ normalized all $ref """ + + class Disp(Dispatcher): pass + + def __init__(self, base_url): + self.base_url = base_url + + @Disp.register([Schema, Parameter, Response, PathItem]) + def _resolve(self, path, obj, _): + obj.update_field('$ref', normalize_jr(getattr(obj, '$ref'), self.base_url)) diff --git a/pyswagger/scanner/v2_0/resolve.py b/pyswagger/scanner/v2_0/resolve.py index 5350b1e..2679b20 100644 --- a/pyswagger/scanner/v2_0/resolve.py +++ b/pyswagger/scanner/v2_0/resolve.py @@ -1,79 +1,29 @@ from __future__ import absolute_import +from ...errs import CycleDetectionError from ...scan import Dispatcher -from ...spec.v2_0.parser import ( - SchemaContext, - ParameterContext, - ResponseContext, - PathItemContext - ) -from ...spec.v2_0.objects import ( - Schema, - Parameter, - Response, - PathItem, - ) -from ...utils import normalize_jr - - -def is_resolved(obj): - return getattr(obj, '$ref') == None or obj.ref_obj != None - -def _resolve(obj, app, parser): - if is_resolved(obj): - return - - r = getattr(obj, '$ref') - ro = app.resolve(normalize_jr(r, app.url), parser) - - if not ro: - raise ReferenceError('Unable to resolve: {0}'.format(r)) - if ro.__class__ != obj.__class__: - raise TypeError('Referenced Type mismatch: {0}'.format(r)) - - obj.update_field('ref_obj', ro) - obj.update_field('norm_ref', normalize_jr(r, app.url)) - -def _merge(obj, app, ctx): - """ resolve $ref as ref_obj, and merge ref_obj to self. - This operation should be carried in a cascade manner. - """ - - cur = obj - to_resolve = [] - while not is_resolved(cur): - _resolve(cur, app, ctx) - - to_resolve.append(cur) - cur = cur.ref_obj if cur.ref_obj else cur - - while (len(to_resolve)): - o = to_resolve.pop() - o.merge(o.ref_obj, ctx, exclude=['$ref']) - +from ...spec.v2_0.parser import SchemaContext +from ...spec.v2_0.objects import Schema class Resolve(object): """ pre-resolve '$ref' """ class Disp(Dispatcher): pass - @Disp.register([Schema]) def _schema(self, _, obj, app): - _resolve(obj, app, SchemaContext) + if obj.ref_obj: + return - @Disp.register([Parameter]) - def _parameter(self, _, obj, app): - _resolve(obj, app, ParameterContext) + r = getattr(obj, '$ref') + if not r: + return - @Disp.register([Response]) - def _response(self, _, obj, app): - _resolve(obj, app, ResponseContext) + ro = app.resolve(r, SchemaContext) + if not ro: + raise ReferenceError('Unable to resolve: {0}'.format(r)) + if ro.__class__ != obj.__class__: + raise TypeError('Referenced Type mismatch: {0}'.format(r)) - @Disp.register([PathItem]) - def _path_item(self, _, obj, app): + obj.update_field('ref_obj', ro) - # $ref in PathItem is 'merge', not 'replace' - # we need to merge properties of others if missing - # in current object. - _merge(obj, app, PathItemContext) diff --git a/pyswagger/spec/v1_2/parser.py b/pyswagger/spec/v1_2/parser.py index bea185d..c23532b 100644 --- a/pyswagger/spec/v1_2/parser.py +++ b/pyswagger/spec/v1_2/parser.py @@ -19,6 +19,8 @@ Resource, Info, ResourceList) +from ...consts import private +from ...utils import url_dirname, url_join class ScopeContext(Context): @@ -159,15 +161,29 @@ class ResourceListContext(Context): def __init__(self, parent, backref): super(ResourceListContext, self).__init__(parent, backref) - def parse(self, getter, obj): + def parse(self, obj, root_url, resolver, getter): super(ResourceListContext, self).parse(obj=obj) + resources = [] + if private.SCHEMA_APIS in obj: + if isinstance(obj[private.SCHEMA_APIS], list): + for api in obj[private.SCHEMA_APIS]: + resources.append(api[private.SCHEMA_PATH]) + else: + raise TypeError('Invalid type of apis: ' + type(obj[private.SCHEMA_APIS])) + base = url_dirname(root_url) + urls = zip( + map(lambda u: url_join(base, u[1:]), resources), + map(lambda u: u[1:], resources) + ) + # replace each element in 'apis' with Resource self._obj['apis'] = {} # get into resource object - for obj, name in getter: + for url, name in urls: # here we assume Resource is always a dict self._obj['apis'][name] = {} + res = resolver.resolve(url, getter) with ResourceContext(self._obj['apis'], name) as ctx: - ctx.parse(obj=obj) + ctx.parse(obj=res) diff --git a/pyswagger/spec/v2_0/objects.py b/pyswagger/spec/v2_0/objects.py index c38bc6f..e9190b5 100644 --- a/pyswagger/spec/v2_0/objects.py +++ b/pyswagger/spec/v2_0/objects.py @@ -1,6 +1,6 @@ from __future__ import absolute_import from ..base import BaseObj, FieldMeta -from ...utils import deref +from ...utils import final from ...io import SwaggerRequest, SwaggerResponse from ...primitives import Array import six @@ -85,7 +85,6 @@ class Schema(six.with_metaclass(FieldMeta, BaseSchema)): # pyswagger only 'ref_obj': None, 'final': None, - 'norm_ref': None, 'name': None, } @@ -177,9 +176,7 @@ class Parameter(six.with_metaclass(FieldMeta, BaseSchema)): } __internal_fields__ = { - # pyswagger only - 'ref_obj': None, - 'norm_ref': None, + 'final': None, } def _prim_(self, v, prim_factory): @@ -216,8 +213,7 @@ class Response(six.with_metaclass(FieldMeta, BaseObj_v2_0)): } __internal_fields__ = { - 'ref_obj': None, - 'norm_ref': None, + 'final': None, } @@ -281,7 +277,7 @@ def _convert_parameter(p): names.append(p.name) for p in self.parameters: - _convert_parameter(deref(p)) + _convert_parameter(final(p)) # check for unknown parameter unknown = set(six.iterkeys(k)) - set(names) @@ -310,11 +306,6 @@ class PathItem(six.with_metaclass(FieldMeta, BaseObj_v2_0)): 'parameters': [], } - __internal_fields__ = { - 'ref_obj': None, - 'norm_ref': None, - } - class SecurityScheme(six.with_metaclass(FieldMeta, BaseObj_v2_0)): """ Security Scheme Object diff --git a/pyswagger/tests/data/v2_0/ex/reuse/definitions/definitions/models.json b/pyswagger/tests/data/v2_0/ex/reuse/definitions/definitions/models.json new file mode 100644 index 0000000..1e2a8ed --- /dev/null +++ b/pyswagger/tests/data/v2_0/ex/reuse/definitions/definitions/models.json @@ -0,0 +1,27 @@ +{ + "models": { + "Model": { + "description": "Another simple model", + "type": "object", + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "tag": { + "description": "a complex, shared property. Note the absolute reference", + "$ref": "models.json#/models/Tag" + } + } + }, + "Tag": { + "description": "A tag entity in the system", + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + } +} diff --git a/pyswagger/tests/data/v2_0/ex/reuse/definitions/models.json b/pyswagger/tests/data/v2_0/ex/reuse/definitions/models.json new file mode 100644 index 0000000..69a341a --- /dev/null +++ b/pyswagger/tests/data/v2_0/ex/reuse/definitions/models.json @@ -0,0 +1,30 @@ +{ + "models": { + "Model": { + "description": "A simple model", + "type": "object", + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "tag": { + "description": "a complex, shared property. Note the absolute reference", + "$ref": "models.json#/models/Tag" + } + } + }, + "Tag": { + "description": "A tag entity in the system", + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + }, + "QQ": { + "$ref":"definitions/models.json#/models/Model" + } + } +} diff --git a/pyswagger/tests/data/v2_0/ex/reuse/operations.json b/pyswagger/tests/data/v2_0/ex/reuse/operations.json new file mode 100644 index 0000000..f3f3ed5 --- /dev/null +++ b/pyswagger/tests/data/v2_0/ex/reuse/operations.json @@ -0,0 +1,23 @@ +{ + "health": { + "get": { + "tags": [ + "admin" + ], + "summary": "Returns server health information", + "operationId": "getHealth", + "produces": [ + "application/json" + ], + "parameters": [], + "responses": { + "200": { + "description": "Health information from the server", + "schema": { + "$ref": "definitions/models.json#/models/Tag" + } + } + } + } + } +} diff --git a/pyswagger/tests/data/v2_0/ex/reuse/parameters/parameters.json b/pyswagger/tests/data/v2_0/ex/reuse/parameters/parameters.json new file mode 100644 index 0000000..872212f --- /dev/null +++ b/pyswagger/tests/data/v2_0/ex/reuse/parameters/parameters.json @@ -0,0 +1,22 @@ +{ + "query" : { + "skip": { + "name": "skip", + "in": "query", + "description": "Results to skip when paginating through a result set", + "required": false, + "minimum": 0, + "type": "integer", + "format": "int32" + }, + "limit": { + "name": "limit", + "in": "query", + "description": "Maximum number of results to return", + "required": false, + "minimum": 0, + "type": "integer", + "format": "int32" + } + } +} diff --git a/pyswagger/tests/data/v2_0/ex/reuse/responses.json b/pyswagger/tests/data/v2_0/ex/reuse/responses.json new file mode 100644 index 0000000..154b679 --- /dev/null +++ b/pyswagger/tests/data/v2_0/ex/reuse/responses.json @@ -0,0 +1,8 @@ +{ + "NotFoundError": { + "description": "Entity not found", + "schema": { + "$ref": "definitions/models.json#/models/Model" + } + } +} diff --git a/pyswagger/tests/data/v2_0/ex/reuse/swagger.json b/pyswagger/tests/data/v2_0/ex/reuse/swagger.json new file mode 100644 index 0000000..8963a2c --- /dev/null +++ b/pyswagger/tests/data/v2_0/ex/reuse/swagger.json @@ -0,0 +1,65 @@ +{ + "swagger":"2.0", + "host":"test.com", + "basePath":"/v1", + "produces":[ + "application/json" + ], + "consumes":[ + "application/json" + ], + "schemes":[ + "http", + "https" + ], + "paths":{ + "/pets":{ + "get":{ + "description":"Returns all pets from the system that the user has access to", + "produces":[ + "application/json" + ], + "parameters":[ + { + "$ref":"parameters/parameters.json#/query/skip" + }, + { + "$ref":"parameters/parameters.json#/query/limit" + }, + { + "in":"query", + "name":"type", + "description":"the types of pet to return", + "required":false, + "type":"string" + } + ], + "responses":{ + "200":{ + "description":"A list of pets.", + "schema":{ + "type":"array", + "items":{ + "$ref":"#/definitions/User" + } + } + }, + "400":{ + "$ref":"responses.json#/NotFoundError" + } + } + } + }, + "/health":{ + "$ref":"operations.json#/health" + } + }, + "definitions":{ + "User":{ + "$ref":"definitions/models.json#/models/Model" + }, + "QQ":{ + "$ref":"definitions/models.json#/models/QQ" + } + } +} diff --git a/pyswagger/tests/data/v2_0/resolve/other/swagger.json b/pyswagger/tests/data/v2_0/resolve/other/swagger.json index c4a977e..9cb286f 100644 --- a/pyswagger/tests/data/v2_0/resolve/other/swagger.json +++ b/pyswagger/tests/data/v2_0/resolve/other/swagger.json @@ -39,7 +39,7 @@ }, "responses":{ "r1":{ - "description":"void" + "description":"void, r1" } } -} \ No newline at end of file +} diff --git a/pyswagger/tests/test_utils.py b/pyswagger/tests/test_utils.py index 69b167b..2859654 100644 --- a/pyswagger/tests/test_utils.py +++ b/pyswagger/tests/test_utils.py @@ -124,10 +124,7 @@ def test_jr_split(self): '', '#')) def test_cycle_guard(self): - def my_id(obj): - return obj - - c = utils.CycleGuard(identity_hook=my_id) + c = utils.CycleGuard() c.update(1) self.assertRaises(errs.CycleDetectionError, c.update, 1) @@ -189,7 +186,7 @@ def test_diff(self): # test include self.assertEqual(sorted(utils._diff_(dict3, dict4, include=['a'])), sorted([ - ('a/a', 1, 2), ('a/b', 3, 2), ('a/c', 4, 5) + ('a/a', 1, 2) ])) # test exclude self.assertEqual(sorted(utils._diff_(dict3, dict4, exclude=['a'])), sorted([ @@ -211,6 +208,19 @@ class A(object): pass self.assertEqual(utils.get_or_none(a, 'b', 'c', 'd'), 'test string') self.assertEqual(utils.get_or_none(a, 'b', 'c', 'd', 'e'), None) + def test_url_dirname(self): + """ test url_dirname + """ + self.assertEqual(utils.url_dirname('https://localhost/test/swagger.json'), 'https://localhost/test') + self.assertEqual(utils.url_dirname('https://localhost/test/'), 'https://localhost/test/') + self.assertEqual(utils.url_dirname('https://localhost/test'), 'https://localhost/test') + + def test_url_join(self): + """ test url_join + """ + self.assertEqual(utils.url_join('https://localhost/test', 'swagger.json'), 'https://localhost/test/swagger.json') + self.assertEqual(utils.url_join('https://localhost/test/', 'swagger.json'), 'https://localhost/test/swagger.json') + class WalkTestCase(unittest.TestCase): """ test for walk """ diff --git a/pyswagger/tests/v1_2/test_upgrade.py b/pyswagger/tests/v1_2/test_upgrade.py index 760c5a8..fa4c9cc 100644 --- a/pyswagger/tests/v1_2/test_upgrade.py +++ b/pyswagger/tests/v1_2/test_upgrade.py @@ -91,7 +91,7 @@ def test_operation(self): r = o.responses['default'] self.assertEqual(r.headers, {}) self.assertEqual(r.schema.type, 'array') - self.assertEqual(getattr(r.schema.items, 'norm_ref'), _pf('/definitions/pet!##!Pet')) + self.assertEqual(getattr(r.schema.items, '$ref'), _pf('/definitions/pet!##!Pet')) # createUser o = self.app.root.paths['/api/user'].post @@ -121,7 +121,7 @@ def test_parameter(self): p = [p for p in o.parameters if getattr(p, 'in') == 'body'][0] self.assertEqual(getattr(p, 'in'), 'body') self.assertEqual(p.required, True) - self.assertEqual(getattr(p.schema, 'norm_ref'), _pf('/definitions/pet!##!Pet')) + self.assertEqual(getattr(p.schema, '$ref'), _pf('/definitions/pet!##!Pet')) # form o = self.app.root.paths['/api/pet/uploadImage'].post @@ -163,7 +163,7 @@ def test_model(self): self.assertEqual(p.maximum, 100) p = d.properties['category'] - self.assertEqual(getattr(p, 'norm_ref'), _pf('/definitions/pet!##!Category')) + self.assertEqual(getattr(p, '$ref'), _pf('/definitions/pet!##!Category')) p = d.properties['photoUrls'] self.assertEqual(p.type, 'array') @@ -171,7 +171,7 @@ def test_model(self): p = d.properties['tags'] self.assertEqual(p.type, 'array') - self.assertEqual(getattr(p.items, 'norm_ref'), _pf('/definitions/pet!##!Tag')) + self.assertEqual(getattr(p.items, '$ref'), _pf('/definitions/pet!##!Tag')) p = d.properties['status'] self.assertEqual(p.type, 'string') diff --git a/pyswagger/tests/v2_0/test_conv.py b/pyswagger/tests/v2_0/test_conv.py index fad91a5..e699ca0 100644 --- a/pyswagger/tests/v2_0/test_conv.py +++ b/pyswagger/tests/v2_0/test_conv.py @@ -22,7 +22,7 @@ def test_v2_0(self): # diff for empty list or dict is allowed d = app.dump() - self.assertEqual(sorted(_diff_(origin, d)), sorted([ + self.assertEqual(sorted(_diff_(origin, d, exclude=['$ref'])), sorted([ ('paths/~1pet~1{petId}/get/security/0/api_key', "list", "NoneType"), ('paths/~1store~1inventory/get/parameters', None, None), ('paths/~1store~1inventory/get/security/0/api_key', "list", "NoneType"), @@ -33,7 +33,7 @@ def test_v2_0(self): tmp = {'_tmp_': {}} with SwaggerContext(tmp, '_tmp_') as ctx: ctx.parse(d) - + class Converter_v1_2_TestCase(unittest.TestCase): """ test for convert from 1.2 @@ -64,7 +64,8 @@ def test_items(self): } self.assertEqual(_diff_( expect, - self.app.s('/api/pet/{petId}').patch.responses['default'].schema.items.dump() + self.app.s('/api/pet/{petId}').patch.responses['default'].schema.items.dump(), + exclude=['$ref'], ), []) # enum @@ -248,7 +249,8 @@ def test_operation(self): } self.assertEqual(_diff_( expect, - self.app.s('/api/pet/findByTags').get.responses['default'].dump() + self.app.s('/api/pet/findByTags').get.responses['default'].dump(), + exclude=['$ref'], ), []) def test_property(self): @@ -317,7 +319,8 @@ def test_model(self): d = self.app.resolve('#/definitions/pet:Pet').dump() self.assertEqual(_diff_( expect, - self.app.resolve('#/definitions/pet:Pet').dump() + self.app.resolve('#/definitions/pet:Pet').dump(), + exclude=['$ref'], ), []) def test_info(self): diff --git a/pyswagger/tests/v2_0/test_ex.py b/pyswagger/tests/v2_0/test_ex.py index 949cf55..75f1994 100644 --- a/pyswagger/tests/v2_0/test_ex.py +++ b/pyswagger/tests/v2_0/test_ex.py @@ -1,5 +1,7 @@ from pyswagger import SwaggerApp from ..utils import get_test_data_folder +from ...utils import deref, final +from ...spec.v2_0.parser import PathItemContext import unittest import os import six @@ -47,7 +49,7 @@ def test_full_path_item(self): self.assertTrue('default' in p.get.responses) self.assertTrue('404' in p.get.responses) - another_p = self.app.resolve('file:///full/swagger.json#/paths/~1user') + another_p = self.app.resolve('file:///full/swagger.json#/paths/~1user', PathItemContext) self.assertNotEqual(id(p), id(another_p)) self.assertTrue('default' in another_p.get.responses) self.assertTrue('404' in another_p.get.responses) @@ -58,8 +60,9 @@ def test_full_path_item_url(self): p = self.app.resolve('#/paths/~1full') self.assertEqual(p.get.url, 'test.com/v1/full') - original_p = self.app.resolve('file:///full/swagger.json#/paths/~1user') - self.assertEqual(original_p.get.url, 'test1.com/v2/user') + # only root document would be patched, others are only loaded for reference + original_p = self.app.resolve('file:///full/swagger.json#/paths/~1user', PathItemContext) + self.assertEqual(original_p.get.url, None) def test_partial_path_item(self): """ make sure partial swagger.json with PathItem @@ -68,7 +71,7 @@ def test_partial_path_item(self): p = self.app.resolve('#/paths/~1partial') self.assertEqual(p.get.url, 'test.com/v1/partial') - original_p = self.app.resolve('file:///partial/path_item/swagger.json') + original_p = self.app.resolve('file:///partial/path_item/swagger.json') self.assertEqual(original_p.get.url, None) def test_partial_schema(self): @@ -107,3 +110,47 @@ def test_relative_schema(self): ) app.prepare() + +class ReuseTestCase(unittest.TestCase): + """ test case for 'reuse', lots of partial swagger document + https://github.com/OAI/OpenAPI-Specification/blob/master/guidelines/REUSE.md#guidelines-for-referencing + """ + @classmethod + def setUpClass(kls): + kls.app = SwaggerApp.load( + url='file:///reuse/swagger.json', + url_load_hook=_gen_hook(get_test_data_folder(version='2.0', which='ex')) + ) + kls.app.prepare() + + def test_relative_folder(self): + """ make sure the url prepend on $ref should be + derived from the path of current document + """ + o = deref(self.app.resolve('#/definitions/QQ')) + self.assertEqual(o.description, 'Another simple model') + + def test_relative_parameter(self): + """ make sure parameter from relative $ref + is correctly injected(merged) in 'final' field. + """ + o = final(self.app.s('pets').get.parameters[0]) + self.assertEqual(o.description, 'Results to skip when paginating through a result set') + + def test_relative_response(self): + """ make sure response from relative $ref + is correctly injected(merged) in 'final' field. + """ + o = final(self.app.s('pets').get.responses['400']) + self.assertEqual(o.description, 'Entity not found') + + def test_relative_path_item(self): + """ make sure path-item from relative $ref + is correctly injected(merged). + """ + o1 = self.app.s('health').get + self.assertEqual(o1.summary, 'Returns server health information') + # make sure these objects are not referenced, but copied. + o2 = self.app.resolve('file:///reuse/operations.json#/health') + self.assertNotEqual(id(o1), id(o2), PathItemContext) + diff --git a/pyswagger/tests/v2_0/test_resolve.py b/pyswagger/tests/v2_0/test_resolve.py index f76d574..cd05917 100644 --- a/pyswagger/tests/v2_0/test_resolve.py +++ b/pyswagger/tests/v2_0/test_resolve.py @@ -1,6 +1,7 @@ from pyswagger import SwaggerApp, utils from pyswagger.spec.v2_0 import objects from ..utils import get_test_data_folder +from ...utils import final import unittest import os @@ -29,6 +30,10 @@ def test_path_item(self): self.assertTrue(b.put.description, 'c.put') self.assertTrue(b.post.description, 'd.post') + c = self.app.resolve(utils.jp_compose('/c', '#/paths')) + self.assertTrue(b.put.description, 'c.put') + self.assertTrue(b.post.description, 'd.post') + class ResolveTestCase(unittest.TestCase): """ test for $ref other than PathItem """ @@ -50,13 +55,13 @@ def test_parameter(self): """ make sure $ref to Parameter works """ p = self.app.s('/a').get - self.assertEqual(id(p.parameters[0].ref_obj), id(self.app.resolve('#/parameters/p1'))) + self.assertEqual(final(p.parameters[0]).name, 'p1_d') def test_response(self): """ make sure $ref to Response works """ p = self.app.s('/a').get - self.assertEqual(id(p.responses['default'].ref_obj), id(self.app.resolve('#/responses/r1'))) + self.assertEqual(final(p.responses['default']).description, 'void, r1') def test_raises(self): """ make sure to raise for invalid input """ diff --git a/pyswagger/utils.py b/pyswagger/utils.py index 3269f57..4143680 100644 --- a/pyswagger/utils.py +++ b/pyswagger/utils.py @@ -87,15 +87,13 @@ class CycleGuard(object): """ Guard for cycle detection """ - def __init__(self, identity_hook=id): + def __init__(self): self.__visited = [] - self.__hook = identity_hook def update(self, obj): - i = self.__hook(obj) - if i in self.__visited: - raise CycleDetectionError('Cycle detected: {0}'.format(obj.__repr__())) - self.__visited.append(i) + if obj in self.__visited: + raise CycleDetectionError('Cycle detected: {0}'.format(getattr(obj, '$ref', None))) + self.__visited.append(obj) # TODO: this function and datetime don't handle leap-second. @@ -259,6 +257,9 @@ def deref(obj, guard=None): guard.update(cur) return cur +def final(obj): + return obj.final if getattr(obj, 'final', None) else obj + def get_dict_as_tuple(d): """ get the first item in dict, and return it as tuple. @@ -302,6 +303,29 @@ def normalize_url(url): return url +def url_dirname(url): + """ Return the folder containing the '.json' file + """ + p = six.moves.urllib.parse.urlparse(url) + for e in [private.FILE_EXT_JSON, private.FILE_EXT_YAML]: + if p.path.endswith(e): + return six.moves.urllib.parse.urlunparse( + p[:2]+ + (os.path.dirname(p.path),)+ + p[3:] + ) + return url + +def url_join(url, path): + """ url version of os.path.join + """ + p = six.moves.urllib.parse.urlparse(url) + return six.moves.urllib.parse.urlunparse( + p[:2]+ + (os.path.join(p.path, path),)+ + p[3:] + ) + def normalize_jr(jr, url=None): """ normalize JSON reference, also fix implicit reference of JSON pointer. @@ -328,7 +352,7 @@ def normalize_jr(jr, url=None): if p.scheme == '' and url: p = six.moves.urllib.parse.urlparse(url) # it's the path of relative file - path = six.moves.urllib.parse.urlunparse(p[:2]+(os.path.join(os.path.dirname(p.path), jr),)+p[3:]) + path = six.moves.urllib.parse.urlunparse(p[:2]+(os.path.join(os.path.dirname(p.path), path),)+p[3:]) else: path = url @@ -396,7 +420,6 @@ def walk(start, ofn, cyc=None): return cyc - def _diff_(src, dst, ret=None, jp=None, exclude=[], include=[]): """ compare 2 dict/list, return a list containing json-pointer indicating what's different, and what's diff exactly. @@ -407,10 +430,10 @@ def _diff_(src, dst, ret=None, jp=None, exclude=[], include=[]): - other: (jp, src, dst) """ - def _dict_(src, dst, ret, jp, exc, inc): + def _dict_(src, dst, ret, jp): ss, sd = set(src.keys()), set(dst.keys()) # what's include is prior to what's exclude - si, se = set(inc or []), set(exc or []) + si, se = set(include or []), set(exclude or []) ss, sd = (ss & si, sd & si) if si else (ss, sd) ss, sd = (ss - se, sd - se) if se else (ss, sd) @@ -424,7 +447,7 @@ def _dict_(src, dst, ret, jp, exc, inc): # same key for k in ss & sd: - _diff_(src[k], dst[k], ret, jp_compose(k, base=jp)) + _diff_(src[k], dst[k], ret, jp_compose(k, base=jp), exclude, include) def _list_(src, dst, ret, jp): if len(src) < len(dst): @@ -462,7 +485,7 @@ def r(x, y): ss, sd = src, dst for idx, (s, d) in enumerate(zip(src, dst)): - _diff_(s, d, ret, jp_compose(str(idx), base=jp)) + _diff_(s, d, ret, jp_compose(str(idx), base=jp), exclude, include) ret = [] if ret == None else ret jp = '' if jp == None else jp @@ -471,7 +494,7 @@ def r(x, y): if not isinstance(dst, dict): ret.append((jp, type(src).__name__, type(dst).__name__,)) else: - _dict_(src, dst, ret, jp, exclude, include) + _dict_(src, dst, ret, jp) elif isinstance(src, list): if not isinstance(dst, list): ret.append((jp, type(src).__name__, type(dst).__name__,))