AppEngine Example

Contributed by @hakunin, to all AppEngine lovers.

This is roughly how I use jsonpickle on AppEngine to move my data in JSON format from local machine to appengine and back.

Notice how I use low level Entity, which while not being documented is easy to work with when your model classes don't use the same property types, or you just need to migrate data without touching it.

Please note that I am not using AppEngine's keys, nor am I preserving them here. They have encoded the namespace name in them and I want to be able to change that. That to create a copy of user's data, perform migrations on them and then switch user's namespace to the migrated one. (and other cool stuff, especially with client side apps that support offline mode)

from app.base_controller import *

import os

from google.appengine.api import namespace_manager
from google.appengine.api.datastore import Query
from google.appengine.api.datastore import Put
from google.appengine.api.datastore import Entity
from google.appengine.api.datastore_types import Text
from google.appengine.ext import db as Db

import json as JSON
import jsonpickle
import base64


class TextReduceHandler(jsonpickle.handlers.BaseHandler):

  def flatten(self, obj, data):
    pickler = self._base
    if not pickler.unpicklable:
      return unicode(obj)
    _, args = obj.__reduce__()
    cls = args[1]
    args = [base64.b64encode(args[2].encode('utf-8'))]
    data['__reduce__'] = (pickler.flatten(cls), args)
    return data

  def restore(self, obj):
    cls, args = obj['__reduce__']
    value = base64.b64decode(args[0]).decode('utf-8')
    unpickler = self._base
    cls = unpickler.restore(cls)
    params = map(unpickler.restore, args[1:])
    params = (value,) + tuple(params)
    return Text(params[0])

jsonpickle.handlers.registry.register(Text, TextReduceHandler)



class ImportExport:

  def export(self, namespace):
    exported = {}
    if namespace == '<ALL>':
      exported = self.export_all()
    else:
      exported[namespace] = self.export_ns(namespace)

    return jsonpickle.encode(exported)
  
  def export_all(self):
    exported = {}
    q = Db.GqlQuery("SELECT * FROM __namespace__")
    for p in q.fetch(100):
      namespace = p.namespace_name
      if namespace != '' and namespace[0] == '_': continue
      exported[namespace] = self.export_ns(namespace)

    return exported

  def export_ns(self, namespace):
    namespace_manager.set_namespace(namespace)
    namespace_kinds = {}

    for kind in Db.GqlQuery("SELECT * FROM __kind__"):
      if kind.kind_name[0] == '_': continue
      entities = Query(kind.kind_name).Run()
      records = []
      for entity in entities:
        row = {}
        for k in entity:
          row[k] = entity[k]
        records.append(row)
      namespace_kinds[kind.kind_name] = records

    return namespace_kinds


  def import_json(self, json):
    namespaces = jsonpickle.decode(json)
    for namespace, entities in namespaces.iteritems():
      if namespace == '' or namespace[0] != '_':
        namespace_manager.set_namespace(namespace)
        for entity_kind, rows in entities.iteritems():
          for row in rows:
            entity = Entity(entity_kind)
            for k in row:
              entity[k] = row[k]
            Put(entity)


  def clear_local(self):
    out = ''
    q = Db.GqlQuery("SELECT * FROM __namespace__")
    for p in q.fetch(100):
      namespace_manager.set_namespace(p.namespace_name)
      out += '\nclear namespace: '+ p.namespace_name

      for kind in Db.GqlQuery("SELECT * FROM __kind__"):
        if kind.kind_name[0] != '_':
          keys = Db.GqlQuery("SELECT __key__ FROM %s" % kind.kind_name)
          out += '\nclear kind: '+ kind.kind_name
          Db.delete(keys)