From 568dd07a98f9817572ac75c5584e7d1fea4b00f5 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Tue, 9 Jun 2015 15:45:18 +0100 Subject: [PATCH 01/24] differentiate updates from inserts to allow for updates to use the syntax --- django_mongodb_engine/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_mongodb_engine/base.py b/django_mongodb_engine/base.py index 4a580bf0..fcd500bd 100644 --- a/django_mongodb_engine/base.py +++ b/django_mongodb_engine/base.py @@ -38,6 +38,7 @@ class DatabaseFeatures(NonrelDatabaseFeatures): supports_microsecond_precision = False supports_long_model_names = False + distinguishes_insert_from_update = True class DatabaseOperations(NonrelDatabaseOperations): From 075608dcc3fea7f5f4635ae41ff12212c543eac7 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Tue, 9 Jun 2015 21:19:08 +0100 Subject: [PATCH 02/24] added folders to gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index a97178f5..ec7a437c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,7 @@ dist/ .tox MANIFEST + + +.env +.idea \ No newline at end of file From f3a221546a864f25952233bfdfb20e7a4266c026 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Tue, 16 Jun 2015 21:51:05 +0100 Subject: [PATCH 03/24] fixed pymongo version to be > 3.0 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index d8d0155b..cc8dbaed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pymongo +pymongo>=3.0 https://github.com/django-nonrel/djangotoolbox/tarball/master https://github.com/django-nonrel/django/tarball/nonrel-1.5 https://github.com/django-nonrel/django-dbindexer/tarball/master diff --git a/setup.py b/setup.py index ca163672..5e9ada94 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ license='2-clause BSD', description=DESCRIPTION, long_description=LONG_DESCRIPTION, - install_requires=['pymongo', 'djangotoolbox>=1.6.0'], + install_requires=['pymongo>=3.0', 'djangotoolbox>=1.6.0'], packages=find_packages(exclude=['tests', 'tests.*']), zip_safe=False, classifiers=[ From 9fa43d4d93ef8b44423e4a20f7727cfea514c75b Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Tue, 16 Jun 2015 21:52:10 +0100 Subject: [PATCH 04/24] removed deprecated MongoReplicaSetClient and all now unused connection parameters --- django_mongodb_engine/base.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/django_mongodb_engine/base.py b/django_mongodb_engine/base.py index 4a580bf0..ee83a6ed 100644 --- a/django_mongodb_engine/base.py +++ b/django_mongodb_engine/base.py @@ -240,24 +240,19 @@ def pop(name, default=None): warnings.warn("slave_okay has been deprecated. " "Please use read_preference instead.") - if replicaset: - connection_class = MongoReplicaSetClient - else: - connection_class = MongoClient + + conn_options = dict( host=host, port=int(port), - max_pool_size=None, document_class=dict, - tz_aware=False, - _connect=True, - auto_start_request=True + tz_aware=False ) conn_options.update(options) try: - self.connection = connection_class(**conn_options) + self.connection = MongoClient(**conn_options) self.database = self.connection[db_name] except TypeError: exc_info = sys.exc_info() From 681b712ca91a2ac02b5ac38ef879bb2d801997bf Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Tue, 16 Jun 2015 21:53:10 +0100 Subject: [PATCH 05/24] removed the use of the fields parameter which has now been changed to projection since pymongo 3.0 --- django_mongodb_engine/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_mongodb_engine/compiler.py b/django_mongodb_engine/compiler.py index a26c0af1..aac1c596 100644 --- a/django_mongodb_engine/compiler.py +++ b/django_mongodb_engine/compiler.py @@ -136,7 +136,7 @@ def get_cursor(self): return [] fields = get_selected_fields(self.query) - cursor = self.collection.find(self.mongo_query, fields=fields) + cursor = self.collection.find(self.mongo_query, fields) if self.ordering: cursor.sort(self.ordering) if self.query.low_mark > 0: From 292e74c557ee12f1066be05ea0493125ebb20418 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Tue, 16 Jun 2015 21:53:59 +0100 Subject: [PATCH 06/24] removed use of the safe option which is now no longer used. All connections are safe by default and have been since the introduction of the MongoClient in pymongo 2.4 --- tests/settings/settings_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/settings/settings_base.py b/tests/settings/settings_base.py index f6c1b4ef..3482d484 100644 --- a/tests/settings/settings_base.py +++ b/tests/settings/settings_base.py @@ -2,7 +2,7 @@ 'default': { 'ENGINE': 'django_mongodb_engine', 'NAME': 'test', - 'OPTIONS': {'OPERATIONS': {'safe': True}}, + 'OPTIONS': {'OPERATIONS': {}}, }, 'other': { 'ENGINE': 'django_mongodb_engine', From 5b4e11dd93d8f8c06f76ac34c74bb0fd57557cf5 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Wed, 17 Jun 2015 10:39:56 +0100 Subject: [PATCH 07/24] added fixture teardown override to wipe out all collections. The django built-in fixture teardown tries to delete all records from the collections, however this breaks on the capped collections. --- tests/mongodb/tests.py | 6 +----- tests/utils.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/mongodb/tests.py b/tests/mongodb/tests.py index 715ec5bc..32fc1ee0 100644 --- a/tests/mongodb/tests.py +++ b/tests/mongodb/tests.py @@ -3,15 +3,12 @@ from django.core.management import call_command from django.contrib.sites.models import Site -from django.db import connection, connections +from django.db import connection from django.db.utils import DatabaseError, IntegrityError from django.db.models import Q - from gridfs import GridOut from pymongo import ASCENDING, DESCENDING - from django_mongodb_engine.base import DatabaseWrapper - from models import * from utils import * @@ -57,7 +54,6 @@ def test_generic_field(self): def test_databasewrapper_api(self): from pymongo.mongo_client import MongoClient from pymongo.database import Database - from pymongo.collection import Collection from random import shuffle if settings.DEBUG: diff --git a/tests/utils.py b/tests/utils.py index fada71c8..cdfb2f1d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -4,7 +4,6 @@ from django.test import TestCase from django.utils.unittest import skip - class TestCase(TestCase): def setUp(self): @@ -12,6 +11,19 @@ def setUp(self): if getattr(settings, 'TEST_DEBUG', False): settings.DEBUG = True + + def _fixture_teardown(self): + from django.db import connections + for connection_name in connections: + + db_wrapper = connections[connection_name] + + for collection_name in db_wrapper.database.collection_names(include_system_collections=False): + db_wrapper.get_collection(collection_name).drop() + + + + def assertEqualLists(self, a, b): self.assertEqual(list(a), list(b)) From 69e028fbb074f2be8d94b57eda79f532128d25b3 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Wed, 17 Jun 2015 11:09:05 +0100 Subject: [PATCH 08/24] changed the flush command to not try and delete/remove records from capped collections --- django_mongodb_engine/base.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/django_mongodb_engine/base.py b/django_mongodb_engine/base.py index ee83a6ed..319ace8c 100644 --- a/django_mongodb_engine/base.py +++ b/django_mongodb_engine/base.py @@ -60,11 +60,20 @@ def sql_flush(self, style, tables, sequence_list, allow_cascade=False): drop all `tables`. No SQL in MongoDB, so just clear all tables here and return an empty list. """ + + + for table in tables: if table.startswith('system.'): # Do not try to drop system collections. continue - self.connection.database[table].remove() + + collection = self.connection.database[table] + options = collection.options() + + if not options.get('capped', False): + collection.delete_many({}) + return [] def validate_autopk_value(self, value): From b02729cfeca1fd77057e0b6878e332607c77ad83 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Wed, 17 Jun 2015 11:09:35 +0100 Subject: [PATCH 09/24] removed the fixture teardown override --- tests/utils.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/utils.py b/tests/utils.py index cdfb2f1d..51505a06 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -11,19 +11,6 @@ def setUp(self): if getattr(settings, 'TEST_DEBUG', False): settings.DEBUG = True - - def _fixture_teardown(self): - from django.db import connections - for connection_name in connections: - - db_wrapper = connections[connection_name] - - for collection_name in db_wrapper.database.collection_names(include_system_collections=False): - db_wrapper.get_collection(collection_name).drop() - - - - def assertEqualLists(self, a, b): self.assertEqual(list(a), list(b)) From 236b1521c2696583aaad78410ccdb55e776d37a4 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Wed, 17 Jun 2015 11:10:08 +0100 Subject: [PATCH 10/24] clear out any existing objects in the setup --- tests/lookup/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/lookup/tests.py b/tests/lookup/tests.py index 12e7cf63..46d608af 100644 --- a/tests/lookup/tests.py +++ b/tests/lookup/tests.py @@ -18,6 +18,11 @@ class LookupTests(TestCase): def setUp(self): + + Author.objects.all().delete() + Article.objects.all().delete() + Tag.objects.all().delete() + # Create a few Authors. self.au1 = Author(name='Author 1') self.au1.save() From 8bc14d4e5486a41d2e3f0c7ab415c6afe9042f4e Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Wed, 17 Jun 2015 11:10:30 +0100 Subject: [PATCH 11/24] removed use of slave_okay option when creating connection --- tests/mongodb/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/mongodb/tests.py b/tests/mongodb/tests.py index 32fc1ee0..2e2548e3 100644 --- a/tests/mongodb/tests.py +++ b/tests/mongodb/tests.py @@ -3,13 +3,14 @@ from django.core.management import call_command from django.contrib.sites.models import Site -from django.db import connection from django.db.utils import DatabaseError, IntegrityError from django.db.models import Q from gridfs import GridOut from pymongo import ASCENDING, DESCENDING from django_mongodb_engine.base import DatabaseWrapper from models import * +from pymongo.collection import Collection + from utils import * @@ -199,7 +200,6 @@ class foodict(dict): with self.custom_database_wrapper({ 'OPTIONS': { - 'SLAVE_OKAY': True, 'TZ_AWARE': True, 'DOCUMENT_CLASS': foodict, }}) as connection: From 495b82d813232b65a2df5c98002836c3bea8e6f4 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Wed, 17 Jun 2015 12:06:01 +0100 Subject: [PATCH 12/24] removed use of slave_okay --- django_mongodb_engine/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django_mongodb_engine/utils.py b/django_mongodb_engine/utils.py index 6974936b..b8a6a85c 100644 --- a/django_mongodb_engine/utils.py +++ b/django_mongodb_engine/utils.py @@ -70,8 +70,7 @@ def log(self, op, duration, args, kwargs=None): logger.debug(msg, extra={'duration': duration}) def find(self, *args, **kwargs): - if not 'slave_okay' in kwargs and self.collection.slave_okay: - kwargs['slave_okay'] = True + return DebugCursor(self, self.collection, *args, **kwargs) def logging_wrapper(method): From 1573e06524567d34aaf3ad64f0acd462db77cec7 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Wed, 17 Jun 2015 12:08:50 +0100 Subject: [PATCH 13/24] added some clean up in the setup to remove any old docs between tests --- tests/contrib/tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/contrib/tests.py b/tests/contrib/tests.py index 179bb04f..da53e39a 100644 --- a/tests/contrib/tests.py +++ b/tests/contrib/tests.py @@ -121,6 +121,9 @@ def test_map_reduce_with_custom_primary_key(self, inline=False): class RawQueryTests(TestCase): def setUp(self): + + MapReduceModel.objects.all().delete() + for i in xrange(10): MapReduceModel.objects.create(n=i, m=i * 2) @@ -156,6 +159,10 @@ def test_raw_update(self): # TODO: Line breaks. class FullTextTest(TestCase): + def setUp(self): + Post.objects.all().delete() + + def test_simple_fulltext(self): blog = Post(content="simple, full text.... search? test") blog.save() From 57cc51dee1852668da6d8d943aeb4ae3cbdbed06 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Wed, 17 Jun 2015 12:10:02 +0100 Subject: [PATCH 14/24] removed unique constraint test as this will now fail due to safe reads - you can't do the seconds create. Added some data cleanup in the test setups. Made the connection args test more specific --- tests/mongodb/tests.py | 77 ++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/tests/mongodb/tests.py b/tests/mongodb/tests.py index 2e2548e3..2dcbcf48 100644 --- a/tests/mongodb/tests.py +++ b/tests/mongodb/tests.py @@ -3,13 +3,15 @@ from django.core.management import call_command from django.contrib.sites.models import Site +from django.db import connection from django.db.utils import DatabaseError, IntegrityError from django.db.models import Q +from encodings.big5 import codec from gridfs import GridOut from pymongo import ASCENDING, DESCENDING from django_mongodb_engine.base import DatabaseWrapper from models import * -from pymongo.collection import Collection + from utils import * @@ -56,16 +58,16 @@ def test_databasewrapper_api(self): from pymongo.mongo_client import MongoClient from pymongo.database import Database from random import shuffle + from pymongo.collection import Collection if settings.DEBUG: - from django_mongodb_engine.utils import \ - CollectionDebugWrapper as Collection + from django_mongodb_engine.utils import CollectionDebugWrapper as Collection + for wrapper in [connection, DatabaseWrapper(connection.settings_dict)]: calls = [ - lambda: self.assertIsInstance(wrapper.get_collection('foo'), - Collection), + lambda: self.assertIsInstance(wrapper.get_collection('foo'), Collection), lambda: self.assertIsInstance(wrapper.database, Database), lambda: self.assertIsInstance(wrapper.connection, MongoClient), ] @@ -176,6 +178,10 @@ def test_issue_89(self): class DatabaseOptionTests(TestCase): """Tests for MongoDB-specific database options.""" + def setUp(self): + Post.objects.all().delete() + RawModel.objects.all().delete() + class custom_database_wrapper(object): def __init__(self, settings, **kwargs): @@ -190,7 +196,7 @@ def __enter__(self): return self.new_wrapper def __exit__(self, *exc_info): - self.new_wrapper.connection.disconnect() + self.new_wrapper.connection.close() connections._connections.default = self._old_connection def test_pymongo_connection_args(self): @@ -198,24 +204,25 @@ def test_pymongo_connection_args(self): class foodict(dict): pass + tz_aware = True + document_class = foodict + with self.custom_database_wrapper({ 'OPTIONS': { 'TZ_AWARE': True, 'DOCUMENT_CLASS': foodict, }}) as connection: - for name, value in connection.settings_dict[ - 'OPTIONS'].iteritems(): - name = '_Connection__%s' % name.lower() - if name not in connection.connection.__dict__: - # slave_okay was moved into BaseObject in PyMongo 2.0. - name = name.replace('Connection', 'BaseObject') - if name not in connection.connection.__dict__: - # document_class was moved into MongoClient in PyMongo 2.4. - name = name.replace('BaseObject', 'MongoClient') - self.assertEqual(connection.connection.__dict__[name], value) + + codec_options = connection.connection.codec_options + + self.assertEqual(codec_options.tz_aware, tz_aware) + self.assertEqual(codec_options.document_class, document_class) + + def test_operation_flags(self): def test_setup(flags, **method_kwargs): + cls_code = [ 'from pymongo.collection import Collection', 'class Collection(Collection):', @@ -233,43 +240,30 @@ def test_setup(flags, **method_kwargs): exec '\n'.join(cls_code) in locals() options = {'OPTIONS': {'OPERATIONS': flags}} - with self.custom_database_wrapper(options, - collection_class=Collection): + with self.custom_database_wrapper(options, collection_class=Collection): RawModel.objects.create(raw='foo') update_count = RawModel.objects.update(raw='foo'), \ RawModel.objects.count() RawModel.objects.all().delete() for name in method_kwargs: - self.assertEqual(method_kwargs[name], - Collection._method_kwargs[name]) + self.assertEqual(method_kwargs[name], Collection._method_kwargs[name]) - if Collection._method_kwargs['update'].get('safe'): - self.assertEqual(*update_count) test_setup({}, save={}, update={'multi': True}, remove={}) - test_setup({ - 'safe': True}, - save={'safe': True}, - update={'safe': True, 'multi': True}, - remove={'safe': True}) - test_setup({ - 'delete': {'safe': True}, 'update': {}}, + test_setup({}, + save={}, + update={'multi': True}, + remove={}) + test_setup({'delete': {}, 'update': {}}, save={}, update={'multi': True}, - remove={'safe': True}) - test_setup({ - 'insert': {'fsync': True}, 'delete': {'fsync': True}}, + remove={}) + test_setup({'insert': {'fsync': True}, 'delete': {'fsync': True}}, save={}, update={'multi': True}, remove={'fsync': True}) - def test_unique(self): - with self.custom_database_wrapper({'OPTIONS': {}}): - Post.objects.create(title='a', content='x') - Post.objects.create(title='a', content='y') - self.assertEqual(Post.objects.count(), 1) - self.assertEqual(Post.objects.get().content, 'x') def test_unique_safe(self): Post.objects.create(title='a') @@ -341,9 +335,12 @@ def assertHaveIndex(self, key, **properties): info = get_collection(NewStyleIndexesTestModel).index_information() index_name = '_'.join('%s_%s' % pair for pair in key) default_properties = {'key': self.order_doesnt_matter(key), 'v': 1} + self.assertIn(index_name, info) - self.assertEqual(info[index_name], - dict(default_properties, **properties)) + + for key, value in dict(default_properties, **properties).iteritems(): + self.assertEqual(info[index_name][key], value) + def test_indexes(self): self.assertHaveIndex([('db_index', 1)]) From 801d7bfe74763015b0340cc26f1fb4b3b111bdf0 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Tue, 23 Jun 2015 15:45:22 +0100 Subject: [PATCH 15/24] updated link to new github user (django-nonrel) --- AUTHORS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 7a3bd943..9fb21035 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -23,4 +23,4 @@ Contributions by * Brandon Pedersen (https://github.com/bpedman) (For an up-to-date list of contributors, see -https://github.com/django-mongodb-engine/mongodb-engine/contributors.) +https://github.com/django-nonrel/mongodb-engine/contributors.) From 5edbaefef28b2df6c72d220b367b422f6a8e3b88 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Tue, 23 Jun 2015 16:04:59 +0100 Subject: [PATCH 16/24] merged changes from ajdavis:pymongo-3 branch --- django_mongodb_engine/utils.py | 1 - docs/source/reference/settings.rst | 36 +++++++++++++----------------- tests/mongodb/tests.py | 28 ++++++++++++----------- 3 files changed, 30 insertions(+), 35 deletions(-) diff --git a/django_mongodb_engine/utils.py b/django_mongodb_engine/utils.py index b8a6a85c..77aa120e 100644 --- a/django_mongodb_engine/utils.py +++ b/django_mongodb_engine/utils.py @@ -70,7 +70,6 @@ def log(self, op, duration, args, kwargs=None): logger.debug(msg, extra={'duration': duration}) def find(self, *args, **kwargs): - return DebugCursor(self, self.collection, *args, **kwargs) def logging_wrapper(method): diff --git a/docs/source/reference/settings.rst b/docs/source/reference/settings.rst index e23ca372..e0ff5458 100644 --- a/docs/source/reference/settings.rst +++ b/docs/source/reference/settings.rst @@ -3,9 +3,9 @@ Settings .. TODO fix highlighting -Connection Settings +Client Settings ------------------- -Additional flags may be passed to :class:`pymongo.Connection` using the +Additional flags may be passed to :class:`pymongo.MongoClient` using the ``OPTIONS`` dictionary:: DATABASES = { @@ -14,9 +14,7 @@ Additional flags may be passed to :class:`pymongo.Connection` using the 'NAME' : 'my_database', ... 'OPTIONS' : { - 'slave_okay' : True, - 'tz_aware' : True, - 'network_timeout' : 42, + 'socketTimeoutMS' : 500, ... } } @@ -24,17 +22,17 @@ Additional flags may be passed to :class:`pymongo.Connection` using the All of these settings directly mirror PyMongo settings. In fact, all Django MongoDB Engine does is lower-casing the names before passing the flags to -:class:`~pymongo.Connection`. For a list of possible options head over to the -`PyMongo documentation on connection options`_. +:class:`~pymongo.MongoClient`. For a list of possible options head over to the +`PyMongo documentation on client options`_. .. _operations-setting: -Safe Operations (``getLastError``) ----------------------------------- +Acknowledged Operations +----------------------- Use the ``OPERATIONS`` dict to specify extra flags passed to :meth:`Collection.save `, :meth:`~pymongo.collection.Collection.update` or -:meth:`~pymongo.collection.Collection.remove` (and thus to ``getLastError``): +:meth:`~pymongo.collection.Collection.remove` (and thus included in the write concern): .. code-block:: python @@ -43,11 +41,7 @@ Use the ``OPERATIONS`` dict to specify extra flags passed to ... } -Since any options to ``getLastError`` imply ``safe=True``, -this configuration passes ``safe=True, w=3`` as keyword arguments to each of -:meth:`~pymongo.collection.Collection.save`, -:meth:`~pymongo.collection.Collection.update` and -:meth:`~pymongo.collection.Collection.remove`. + Get a more fine-grained setup by introducing another layer to this dict: @@ -55,9 +49,9 @@ Get a more fine-grained setup by introducing another layer to this dict: 'OPTIONS' : { 'OPERATIONS' : { - 'save' : {'safe' : True}, + 'save' : {'w' : 3}, 'update' : {}, - 'delete' : {'fsync' : True} + 'delete' : {'j' : True} }, ... } @@ -69,10 +63,10 @@ Get a more fine-grained setup by introducing another layer to this dict: "`insert vs. update`" into `save`. -A full list of ``getLastError`` flags may be found in the -`MongoDB documentation `_. +A full list of write concern flags may be found in the +`MongoDB documentation `_. .. _Similar to Django's built-in backends: http://docs.djangoproject.com/en/dev/ref/settings/#std:setting-OPTIONS -.. _PyMongo documentation on connection options: - http://api.mongodb.org/python/current/api/pymongo/connection.html +.. _PyMongo documentation on client options: + http://api.mongodb.org/python/current/api/pymongo/mongo_client.html diff --git a/tests/mongodb/tests.py b/tests/mongodb/tests.py index 40e62554..58418542 100644 --- a/tests/mongodb/tests.py +++ b/tests/mongodb/tests.py @@ -8,7 +8,7 @@ from django.db.models import Q from encodings.big5 import codec from gridfs import GridOut -from pymongo import ASCENDING, DESCENDING +from pymongo import ASCENDING, DESCENDING, ReadPreference, version_tuple as pymongo_version from django_mongodb_engine.base import DatabaseWrapper from models import * @@ -204,20 +204,26 @@ def test_pymongo_connection_args(self): class foodict(dict): pass - tz_aware = True - document_class = foodict - with self.custom_database_wrapper({ 'OPTIONS': { + 'READ_PREFERENCE': ReadPreference.SECONDARY, 'TZ_AWARE': True, 'DOCUMENT_CLASS': foodict, - }}) as connection: + }}) as db: + + connection = db.connection - codec_options = connection.connection.codec_options + if pymongo_version[0] >= 3: + tz_aware = connection.codec_options.tz_aware + document_class = connection.codec_options.document_class + else: + tz_aware = connection.tz_aware + document_class = connection.document_class - self.assertEqual(codec_options.tz_aware, tz_aware) - self.assertEqual(codec_options.document_class, document_class) + self.assertEqual(ReadPreference.SECONDARY, connection.read_preference) + self.assertEqual(True, tz_aware) + self.assertEqual(foodict, document_class) def test_operation_flags(self): @@ -254,11 +260,7 @@ def test_setup(flags, **method_kwargs): test_setup({}, save={}, update={'multi': True}, remove={}) test_setup({}, save={}, update={'multi': True}, remove={}) test_setup({'delete': {}, 'update': {}}, save={}, update={'multi': True}, remove={}) - test_setup({ 'insert': {'fsync': True}, 'delete': {'fsync': True}}, - - save={}, - update={'multi': True}, - remove={'fsync': True}) + test_setup({ 'insert': {'fsync': True}, 'delete': {'fsync': True}}, save={}, update={'multi': True}, remove={'fsync': True}) def test_unique_safe(self): From c0dcc66e137cd1d7b2673d5891727df9e57665c6 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Tue, 23 Jun 2015 17:22:56 +0100 Subject: [PATCH 17/24] removed distinguishes_insert_from_update flag in DatabaseFeatures (wrong pull request). pip install from tarballs --- .travis.yml | 6 +++--- django_mongodb_engine/base.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e0fea76f..4e781c35 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,9 +21,9 @@ matrix: - env: DJANGO_VERSION=1.4 install: - - pip install git+http://github.com/django-nonrel/django@nonrel-$DJANGO_VERSION - - pip install git+http://github.com/django-nonrel/django-dbindexer@master - - pip install git+http://github.com/django-nonrel/djangotoolbox@master + - pip install https://github.com/django-nonrel/django/tarball/nonrel-$DJANGO_VERSION + - pip install https://github.com/django-nonrel/django-dbindexer/tarball/master + - pip install https://github.com/django-nonrel/djangotoolbox/tarball/master - python setup.py install script: cd tests && python runtests.py diff --git a/django_mongodb_engine/base.py b/django_mongodb_engine/base.py index 6c31f01c..2b34bd1c 100644 --- a/django_mongodb_engine/base.py +++ b/django_mongodb_engine/base.py @@ -38,7 +38,7 @@ class DatabaseFeatures(NonrelDatabaseFeatures): supports_microsecond_precision = False supports_long_model_names = False - distinguishes_insert_from_update = True + #distinguishes_insert_from_update = True class DatabaseOperations(NonrelDatabaseOperations): From fef01855d9cc60d1a29f6ddd17e570564e90ef97 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Fri, 26 Jun 2015 12:03:06 +0100 Subject: [PATCH 18/24] reverted change to use tarballs instead of cloning from git --- .travis.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4e781c35..9ad8bc88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,16 +14,18 @@ python: env: - DJANGO_VERSION=1.4 - DJANGO_VERSION=1.5 + - DJANGO_VERSION=1.6 - DJANGO_VERSION=1.6-beta + - DJANGO_VERSION=1.7 matrix: allow_failures: - env: DJANGO_VERSION=1.4 install: - - pip install https://github.com/django-nonrel/django/tarball/nonrel-$DJANGO_VERSION - - pip install https://github.com/django-nonrel/django-dbindexer/tarball/master - - pip install https://github.com/django-nonrel/djangotoolbox/tarball/master + - pip install git+http://github.com/django-nonrel/django@nonrel-$DJANGO_VERSION + - pip install git+http://github.com/django-nonrel/django-dbindexer@master + - pip install git+http://github.com/django-nonrel/djangotoolbox@master - python setup.py install script: cd tests && python runtests.py From af3d08b7bdff4dea14198c10ad3e12dadc566b52 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Fri, 26 Jun 2015 12:17:05 +0100 Subject: [PATCH 19/24] removed 1.7 from test options --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index af0c62b7..86460ec3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,6 @@ env: - DJANGO_VERSION=1.4 - DJANGO_VERSION=1.5 - DJANGO_VERSION=1.6 - - DJANGO_VERSION=1.7 matrix: From 7395f5f1ca3edf91581ba6bf186df95cbcac8584 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Fri, 26 Jun 2015 12:18:21 +0100 Subject: [PATCH 20/24] added 1.7 back in, but allow to fail for now --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 86460ec3..57e8458c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,11 +15,13 @@ env: - DJANGO_VERSION=1.4 - DJANGO_VERSION=1.5 - DJANGO_VERSION=1.6 + - DJANGO_VERSION=1.7 matrix: allow_failures: - env: DJANGO_VERSION=1.4 + - env: DJANGO_VERSION=1.7 install: - pip install git+http://github.com/django-nonrel/django@nonrel-$DJANGO_VERSION From 122e62b6ab143aea9f536e23c37dab88f3c523f9 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Fri, 26 Jun 2015 14:46:17 +0100 Subject: [PATCH 21/24] added can_rollback_ddl=True in DatabaseFeatures to fix issues with atomic() in flush command --- django_mongodb_engine/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_mongodb_engine/base.py b/django_mongodb_engine/base.py index 2b34bd1c..b5696c86 100644 --- a/django_mongodb_engine/base.py +++ b/django_mongodb_engine/base.py @@ -38,7 +38,8 @@ class DatabaseFeatures(NonrelDatabaseFeatures): supports_microsecond_precision = False supports_long_model_names = False - #distinguishes_insert_from_update = True + + can_rollback_ddl = True class DatabaseOperations(NonrelDatabaseOperations): From e6d2b384efa5ada353b1165ed0caf63760ab8da7 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Fri, 26 Jun 2015 16:32:46 +0100 Subject: [PATCH 22/24] support pymongo 2.x --- django_mongodb_engine/base.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/django_mongodb_engine/base.py b/django_mongodb_engine/base.py index b5696c86..a9c57058 100644 --- a/django_mongodb_engine/base.py +++ b/django_mongodb_engine/base.py @@ -76,7 +76,7 @@ def sql_flush(self, style, tables, sequence_list, allow_cascade=False): if not options.get('capped', False): # TODO:Not backwards compatible - collection.delete_many({}) + collection.remove({}) return [] @@ -253,9 +253,6 @@ def pop(name, default=None): warnings.warn("slave_okay has been deprecated. " "Please use read_preference instead.") - - - conn_options = dict( host=host, port=int(port), @@ -264,8 +261,13 @@ def pop(name, default=None): ) conn_options.update(options) + if replicaset: + connection_class = MongoReplicaSetClient + else: + connection_class = MongoClient + try: - self.connection = MongoClient(**conn_options) + self.connection = connection_class(**conn_options) self.database = self.connection[db_name] except TypeError: exc_info = sys.exc_info() From 7faa4a45def0b273eceb7b6a1c622a8d15ac6045 Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Fri, 26 Jun 2015 16:33:23 +0100 Subject: [PATCH 23/24] added environment variables for testing against pymongo 2.8.1 and and pymongo 3.0.2 --- .travis.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 57e8458c..40e990ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,21 +12,28 @@ python: - 2.7 env: - - DJANGO_VERSION=1.4 - - DJANGO_VERSION=1.5 - - DJANGO_VERSION=1.6 - - DJANGO_VERSION=1.7 + - DJANGO_VERSION=1.4 PYMONGO_VERSION=2.8.1 + - DJANGO_VERSION=1.4 PYMONGO_VERSION=3.0.2 + - DJANGO_VERSION=1.5 PYMONGO_VERSION=2.8.1 + - DJANGO_VERSION=1.5 PYMONGO_VERSION=3.0.2 + - DJANGO_VERSION=1.6 PYMONGO_VERSION=2.8.1 + - DJANGO_VERSION=1.6 PYMONGO_VERSION=3.0.2 + - DJANGO_VERSION=1.7 PYMONGO_VERSION=2.8.1 + - DJANGO_VERSION=1.7 PYMONGO_VERSION=3.0.2 matrix: allow_failures: - - env: DJANGO_VERSION=1.4 - - env: DJANGO_VERSION=1.7 + - env: DJANGO_VERSION=1.4 PYMONGO_VERSION=2.8.1 + - env: DJANGO_VERSION=1.4 PYMONGO_VERSION=3.0.2 + - env: DJANGO_VERSION=1.7 PYMONGO_VERSION=2.8.1 + - env: DJANGO_VERSION=1.7 PYMONGO_VERSION=3.0.2 install: - pip install git+http://github.com/django-nonrel/django@nonrel-$DJANGO_VERSION - pip install git+http://github.com/django-nonrel/django-dbindexer@master - pip install git+http://github.com/django-nonrel/djangotoolbox@master + - pip install pymongo==$PYMONGO_VERSION --upgrade - python setup.py install script: cd tests && python runtests.py From 605dfcb14d388901aee302ec3956450d1dab9d8f Mon Sep 17 00:00:00 2001 From: Mark Unsworth Date: Mon, 29 Jun 2015 10:38:16 +0100 Subject: [PATCH 24/24] cleanup based on @aburgel comments --- django_mongodb_engine/base.py | 2 -- requirements.txt | 2 +- setup.py | 2 +- tests/lookup/tests.py | 10 ---------- tests/mongodb/tests.py | 1 - 5 files changed, 2 insertions(+), 15 deletions(-) diff --git a/django_mongodb_engine/base.py b/django_mongodb_engine/base.py index a9c57058..12ddd2e1 100644 --- a/django_mongodb_engine/base.py +++ b/django_mongodb_engine/base.py @@ -74,8 +74,6 @@ def sql_flush(self, style, tables, sequence_list, allow_cascade=False): options = collection.options() if not options.get('capped', False): - - # TODO:Not backwards compatible collection.remove({}) return [] diff --git a/requirements.txt b/requirements.txt index 7dd12409..2d702cc1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -pymongo>=3.0 +pymongo>=2.8 https://github.com/django-nonrel/djangotoolbox/tarball/master https://github.com/django-nonrel/django/tarball/nonrel-1.5 diff --git a/setup.py b/setup.py index 240c3411..d917dca6 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ description=DESCRIPTION, long_description=LONG_DESCRIPTION, - install_requires=['pymongo>=3.0', 'djangotoolbox>=1.6.0'], + install_requires=['pymongo>=2.8', 'djangotoolbox>=1.6.0'], packages=find_packages(exclude=['tests', 'tests.*']), zip_safe=False, diff --git a/tests/lookup/tests.py b/tests/lookup/tests.py index 6fe69a6f..b97a770a 100644 --- a/tests/lookup/tests.py +++ b/tests/lookup/tests.py @@ -2,7 +2,6 @@ from operator import attrgetter from django.core.exceptions import FieldError -from django.db import connection from django.db.utils import DatabaseError from django.test import TestCase, skipUnlessDBFeature @@ -19,15 +18,6 @@ class LookupTests(TestCase): def setUp(self): - Author.objects.all().delete() - Article.objects.all().delete() - Tag.objects.all().delete() - - # Create a few Authors. - Author.objects.all().delete() - Article.objects.all().delete() - Tag.objects.all().delete() - self.au1 = Author(name='Author 1') self.au1.save() self.au2 = Author(name='Author 2') diff --git a/tests/mongodb/tests.py b/tests/mongodb/tests.py index 58418542..1b773b9c 100644 --- a/tests/mongodb/tests.py +++ b/tests/mongodb/tests.py @@ -6,7 +6,6 @@ from django.db import connection from django.db.utils import DatabaseError, IntegrityError from django.db.models import Q -from encodings.big5 import codec from gridfs import GridOut from pymongo import ASCENDING, DESCENDING, ReadPreference, version_tuple as pymongo_version from django_mongodb_engine.base import DatabaseWrapper