diff --git a/.travis.yml b/.travis.yml index 10cf83f6..cffb42b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,11 +3,8 @@ python: 2.7 services: - couchdb env: - - TOX_ENV=py26 - TOX_ENV=py27 - - TOX_ENV=py33 - TOX_ENV=py34 - - TOX_ENV=py33-json - TOX_ENV=py34-json install: - pip install tox diff --git a/ChangeLog.rst b/ChangeLog.rst index a1ab9a53..7cf2f217 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,37 @@ +Version 1.2 (2018-02-09) +------------------------ + +* Fixed some issues relating to usage with Python 3 +* Remove support for Python 2.6 and 3.x with x < 4 +* Fix logging response in query server (fixes #321) +* Fix HTTP authentication password encoding (fixes #302) +* Add missing ``http.Forbidden`` error (fixes #305) +* Show ``doc`` property on ``Row`` string representation +* Add methods for mango queries and indexes +* Allow mango filters in ``_changes`` API + + +Version 1.1 (2016-08-05) +------------------------ + +* Add script to load design documents from disk +* Add methods on ``Server`` for user/session management +* Add microseconds support for DateTimeFields +* Handle changes feed as emitted by CouchBase (fixes #289) +* Support Python 3 in ``couchdb-dump`` script (fixes #296) +* Expand relative URLs from Location headers (fixes #287) +* Correctly handle ``_rev`` fields in mapped documents (fixes #278) + + +Version 1.0.1 (2016-03-12) +-------------------------- + +* Make sure connections are correctly closed on GAE (fixes #224) +* Correctly join path parts in replicate script (fixes #269) +* Fix id and rev for some special documents +* Make it possible to disable SSL verification + + Version 1.0 (2014-11-16) ------------------------ diff --git a/Makefile b/Makefile index eba2386f..f4628527 100644 --- a/Makefile +++ b/Makefile @@ -12,8 +12,8 @@ test3: doc: python setup.py build_sphinx -upload-doc: - python setup.py upload_sphinx +upload-doc: doc + python2 setup.py upload_sphinx coverage: PYTHONPATH=. coverage run couchdb/tests/__main__.py diff --git a/README.rst b/README.rst index ae1e910b..65e0f029 100644 --- a/README.rst +++ b/README.rst @@ -4,6 +4,12 @@ CouchDB-Python Library .. image:: https://travis-ci.org/djc/couchdb-python.svg :target: https://travis-ci.org/djc/couchdb-python +**Note: CouchDB-Python is no longer being maintained. After 8 years of maintaining +CouchDB-Python, I no longer have time to address open issues and new bug reports. +Consider https://github.com/cloudant/python-cloudant as an alternative. +If you're interested in taking over maintenance of CouchDB-Python, please start a +discussion on the mailing list, or open an issue or PR.** + A Python library for working with CouchDB. `Downloads`_ are available via `PyPI`_. Our `documentation`_ is also hosted there. We have a `mailing list`_. @@ -22,11 +28,12 @@ It also provides a couple of command-line tools: Prerequisites: -* simplejson (or Python >= 2.6, which comes with a simplejson-based JSON module in the standard library) -* Python 2.6 or later +* Python 2.7, 3.4 or later * CouchDB 0.10.x or later (0.9.x should probably work, as well) +``simplejson`` will be used if installed. + .. _Downloads: http://pypi.python.org/pypi/CouchDB .. _PyPI: http://pypi.python.org/ -.. _documentation: http://packages.python.org/CouchDB/ +.. _documentation: http://couchdb-python.readthedocs.io/en/latest/ .. _mailing list: http://groups.google.com/group/couchdb-python diff --git a/RELEASING.rst b/RELEASING.rst index 87ba8337..318b7a50 100644 --- a/RELEASING.rst +++ b/RELEASING.rst @@ -4,11 +4,11 @@ Release procedure A list of steps to perform when releasing. * Run tests against latest CouchDB release (ideally also trunk) -* Run tests on different Python versions +* Make sure the version number in setup.py is correct * Update ChangeLog and add a release date, then commit -* Merge changes from default to stable -* Edit setup.cfg (in the tag), remove the egg_info section and commit -* Tag the just-committed changeset, then update to the tag +* Edit setup.cfg, remove the egg_info section and commit +* Tag the just-committed changeset * python setup.py bdist_wheel sdist --formats=gztar upload -* Revert the setup.cfg change. -* Update the version number on the branch to 0.10.1 +* Revert the setup.cfg change +* Update the version number in setup.py +* Upload docs to PyPI with ``make upload-doc`` diff --git a/couchdb/__init__.py b/couchdb/__init__.py index 06edd826..e7f6bbf0 100644 --- a/couchdb/__init__.py +++ b/couchdb/__init__.py @@ -8,7 +8,8 @@ from .client import Database, Document, Server from .http import HTTPError, PreconditionFailed, Resource, \ - ResourceConflict, ResourceNotFound, ServerError, Session, Unauthorized + ResourceConflict, ResourceNotFound, ServerError, Session, \ + Unauthorized, Forbidden try: __version__ = __import__('pkg_resources').get_distribution('CouchDB').version diff --git a/couchdb/client.py b/couchdb/client.py index 6f571e2e..6125e831 100644 --- a/couchdb/client.py +++ b/couchdb/client.py @@ -30,6 +30,8 @@ from inspect import getsource from textwrap import dedent import warnings +import sys +import socket from couchdb import http, json, util @@ -43,7 +45,9 @@ class Server(object): """Representation of a CouchDB server. - >>> server = Server() + >>> server = Server() # connects to the local_server + >>> remote_server = Server('http://example.com:5984/') + >>> secure_remote_server = Server('https://username:password@example.com:5984/') This class behaves like a dictionary of databases. For example, to get a list of database names on the server, you can simply iterate over the @@ -81,6 +85,7 @@ def __init__(self, url=DEFAULT_BASE_URL, full_commit=True, session=None): self.resource = url # treat as a Resource object if not full_commit: self.resource.headers['X-Couch-Full-Commit'] = 'false' + self._version_info = None def __contains__(self, name): """Return whether the server contains a database with the specified @@ -92,7 +97,7 @@ def __contains__(self, name): try: self.resource.head(name) return True - except http.ResourceNotFound: + except (socket.error, http.ResourceNotFound): return False def __iter__(self): @@ -110,7 +115,7 @@ def __nonzero__(self): try: self.resource.head() return True - except: + except (socket.error, http.ResourceNotFound): return False def __bool__(self): @@ -162,6 +167,18 @@ def version(self): status, headers, data = self.resource.get_json() return data['version'] + def version_info(self): + """The version of the CouchDB server as a tuple of ints. + + Note that this results in a request being made only at the first call. + Afterwards the result will be cached. + + :rtype: `tuple(int, int, int)`""" + if self._version_info is None: + version = self.version() + self._version_info = tuple(map(int, version.split('.'))) + return self._version_info + def stats(self, name=None): """Server statistics. @@ -225,6 +242,81 @@ def replicate(self, source, target, **options): status, headers, data = self.resource.post_json('_replicate', data) return data + def add_user(self, name, password, roles=None): + """Add regular user in authentication database. + + :param name: name of regular user, normally user id + :param password: password of regular user + :param roles: roles of regular user + :return: (id, rev) tuple of the registered user + :rtype: `tuple` + """ + user_db = self['_users'] + return user_db.save({ + '_id': 'org.couchdb.user:' + name, + 'name': name, + 'password': password, + 'roles': roles or [], + 'type': 'user', + }) + + def remove_user(self, name): + """Remove regular user in authentication database. + + :param name: name of regular user, normally user id + """ + user_db = self['_users'] + doc_id = 'org.couchdb.user:' + name + del user_db[doc_id] + + def login(self, name, password): + """Login regular user in couch db + + :param name: name of regular user, normally user id + :param password: password of regular user + :return: authentication token + """ + status, headers, _ = self.resource.post_json('_session', { + 'name': name, + 'password': password, + }) + if sys.version_info[0] > 2: + cookie = headers._headers[0][1] + else: + cookie = headers.headers[0].split(';')[0] + pos = cookie.find('=') + return cookie[pos + 1:] + + def logout(self, token): + """Logout regular user in couch db + + :param token: token of login user + :return: True if successfully logout + :rtype: bool + """ + header = { + 'Accept': 'application/json', + 'Cookie': 'AuthSession=' + token, + } + status, _, _ = self.resource.delete_json('_session', headers=header) + return status == 200 + + def verify_token(self, token): + """Verify user token + + :param token: authentication token + :return: True if authenticated ok + :rtype: bool + """ + try: + status, _, _ = self.resource.get_json('_session', { + 'Accept': 'application/json', + 'Cookie': 'AuthSession=' + token, + }) + except http.Unauthorized: + return False + return status == 200 + class Database(object): """Representation of a database on a CouchDB server. @@ -318,7 +410,7 @@ def __nonzero__(self): try: self.resource.head() return True - except: + except http.ResourceNotFound: return False def __bool__(self): @@ -408,7 +500,7 @@ def save(self, doc, **options): If doc has no _id then the server will allocate a random ID and a new document will be created. Otherwise the doc's _id will be used to - identity the document to create or update. Trying to update an existing + identify the document to create or update. Trying to update an existing document with an incorrect _rev will raise a ResourceConflict exception. Note that it is generally better to avoid saving documents with no _id @@ -688,10 +780,80 @@ def put_attachment(self, doc, content, filename=None, content_type=None): }, rev=doc['_rev']) doc['_rev'] = data['rev'] + def find(self, mango_query, wrapper=None): + """Execute a mango find-query against the database. + + Note: only available for CouchDB version >= 2.0.0 + + More information on the `mango_query` structure can be found here: + http://docs.couchdb.org/en/master/api/database/find.html#find-selectors + + >>> server = Server() + >>> db = server.create('python-tests') + >>> db['johndoe'] = dict(type='Person', name='John Doe') + >>> db['maryjane'] = dict(type='Person', name='Mary Jane') + >>> db['gotham'] = dict(type='City', name='Gotham City') + >>> mango = {'selector': {'type': 'Person'}, + ... 'fields': ['name'], + ... 'sort':[{'name': 'asc'}]} + >>> for row in db.find(mango): # doctest: +SKIP + ... print(row['name']) # doctest: +SKIP + John Doe + Mary Jane + >>> del server['python-tests'] + + :param mango_query: a dictionary describing criteria used to select + documents + :param wrapper: an optional callable that should be used to wrap the + resulting documents + :return: the query results as a list of `Document` (or whatever `wrapper` returns) + """ + status, headers, data = self.resource.post_json('_find', mango_query) + return map(wrapper or Document, data.get('docs', [])) + + def explain(self, mango_query): + """Explain a mango find-query. + + Note: only available for CouchDB version >= 2.0.0 + + More information on the `mango_query` structure can be found here: + http://docs.couchdb.org/en/master/api/database/find.html#db-explain + + >>> server = Server() + >>> db = server.create('python-tests') + >>> db['johndoe'] = dict(type='Person', name='John Doe') + >>> db['maryjane'] = dict(type='Person', name='Mary Jane') + >>> db['gotham'] = dict(type='City', name='Gotham City') + >>> mango = {'selector': {'type': 'Person'}, 'fields': ['name']} + >>> db.explain(mango) #doctest: +ELLIPSIS +SKIP + {...} + >>> del server['python-tests'] + + :param mango_query: a `dict` describing criteria used to select + documents + :return: the query results as a list of `Document` (or whatever + `wrapper` returns) + :rtype: `dict` + """ + _, _, data = self.resource.post_json('_explain', mango_query) + return data + + def index(self): + """Get an object to manage the database indexes. + + :return: an `Indexes` object to manage the databes indexes + :rtype: `Indexes` + """ + return Indexes(self.resource('_index')) + + + def query(self, map_fun, reduce_fun=None, language='javascript', wrapper=None, **options): """Execute an ad-hoc query (a "temp view") against the database. + Note: not supported for CouchDB version >= 2.0.0 + >>> server = Server() >>> db = server.create('python-tests') >>> db['johndoe'] = dict(type='Person', name='John Doe') @@ -724,7 +886,7 @@ def query(self, map_fun, reduce_fun=None, language='javascript', :param wrapper: an optional callable that should be used to wrap the result rows :param options: optional query string parameters - :return: the view reults + :return: the view results :rtype: `ViewResults` """ return TemporaryView(self.resource('_temp_view'), map_fun, @@ -933,7 +1095,10 @@ def update_doc(self, name, docid=None, **options): :param name: the name of the update handler function in the format ``designdoc/updatename``. :param docid: optional ID of a document to pass to the update handler. - :param options: optional query string parameters. + :param options: additional (optional) params to pass to the underlying + http resource handler, including ``headers``, ``body``, + and ```path```. Other arguments will be treated as + query string params. See :class:`couchdb.http.Resource` :return: (headers, body) tuple, where headers is a dict of headers returned from the list function and body is a readable file-like instance @@ -948,7 +1113,12 @@ def update_doc(self, name, docid=None, **options): return headers, body def _changes(self, **opts): - _, _, data = self.resource.get('_changes', **opts) + # use streaming `get` and `post` methods + if opts.get('filter') == '_selector': + selector = opts.pop('_selector', None) + _, _, data = self.resource.post('_changes', selector, **opts) + else: + _, _, data = self.resource.get('_changes', **opts) lines = data.iterchunks() for ln in lines: if not ln: # skip heartbeats @@ -967,7 +1137,12 @@ def changes(self, **opts): """ if opts.get('feed') == 'continuous': return self._changes(**opts) - _, _, data = self.resource.get_json('_changes', **opts) + + if opts.get('filter') == '_selector': + selector = opts.pop('_selector', None) + _, _, data = self.resource.post_json('_changes', selector, **opts) + else: + _, _, data = self.resource.get_json('_changes', **opts) return data @@ -1009,7 +1184,8 @@ def id(self): :rtype: basestring """ - return self['_id'] + return self.get('_id') + @property def rev(self): @@ -1017,7 +1193,7 @@ def rev(self): :rtype: basestring """ - return self['_rev'] + return self.get('_rev') class View(object): @@ -1245,7 +1421,7 @@ class Row(dict): """Representation of a row as returned by database views.""" def __repr__(self): - keys = 'id', 'key', 'error', 'value' + keys = 'id', 'key', 'doc', 'error', 'value' items = ['%s=%r' % (k, self[k]) for k in keys if k in self] return '<%s %s>' % (type(self).__name__, ', '.join(items)) @@ -1277,3 +1453,111 @@ def doc(self): doc = self.get('doc') if doc: return Document(doc) + + +class Indexes(object): + """Manage indexes in CouchDB 2.0.0 and later. + + More information here: + http://docs.couchdb.org/en/2.0.0/api/database/find.html#db-index + """ + + def __init__(self, url, session=None): + if isinstance(url, util.strbase): + self.resource = http.Resource(url, session) + else: + self.resource = url + + def __setitem__(self, ddoc_name, index): + """Add an index to the database. + + >>> server = Server() + >>> db = server.create('python-tests') + >>> db['johndoe'] = dict(type='Person', name='John Doe') + >>> db['maryjane'] = dict(type='Person', name='Mary Jane') + >>> db['gotham'] = dict(type='City', name='Gotham City') + >>> idx = db.index() + >>> idx['foo', 'bar'] = [{'type': 'asc'}] #doctest: +SKIP + >>> list(idx) #doctest: +SKIP + [{'ddoc': None, + 'def': {'fields': [{'_id': 'asc'}]}, + 'name': '_all_docs', + 'type': 'special'}, + {'ddoc': '_design/foo', + 'def': {'fields': [{'type': 'asc'}]}, + 'name': 'bar', + 'type': 'json'}] + >>> idx[None, None] = [{'type': 'desc'}] #doctest: +SKIP + >>> list(idx) #doctest: +SKIP, +ELLIPSIS + [{'ddoc': None, + 'def': {'fields': [{'_id': 'asc'}]}, + 'name': '_all_docs', + 'type': 'special'}, + {'ddoc': '_design/...', + 'def': {'fields': [{'type': 'desc'}]}, + 'name': '...', + 'type': 'json'}, + {'ddoc': '_design/foo', + 'def': {'fields': [{'type': 'asc'}]}, + 'name': 'bar', + 'type': 'json'}] + >>> del server['python-tests'] + + :param index: `list` of indexes to create + :param ddoc_name: `tuple` or `list` containing first the name of the + design document, in which the index will be created, + and second name of the index. Both can be `None`. + """ + query = {'index': {'fields': index}} + ddoc, name = ddoc_name # expect ddoc / name to be a slice or list + if ddoc: + query['ddoc'] = ddoc + if name: + query['name'] = name + self.resource.post_json(body=query) + + def __delitem__(self, ddoc_name): + """Remove an index from the database. + + >>> server = Server() + >>> db = server.create('python-tests') + >>> db['johndoe'] = dict(type='Person', name='John Doe') + >>> db['maryjane'] = dict(type='Person', name='Mary Jane') + >>> db['gotham'] = dict(type='City', name='Gotham City') + >>> idx = db.index() + >>> idx['foo', 'bar'] = [{'type': 'asc'}] #doctest: +SKIP + >>> del idx['foo', 'bar'] #doctest: +SKIP + >>> list(idx) #doctest: +SKIP + [{'ddoc': None, + 'def': {'fields': [{'_id': 'asc'}]}, + 'name': '_all_docs', + 'type': 'special'}] + >>> del server['python-tests'] + + :param ddoc: name of the design document containing the index + :param name: name of the index that is to be removed + :return: `dict` containing the `id`, the `name` and the `result` of + creating the index + """ + self.resource.delete_json([ddoc_name[0], 'json', ddoc_name[1]]) + + def _list(self): + _, _, data = self.resource.get_json() + return data + + def __iter__(self): + """Iterate all indexes of the associated database. + + >>> server = Server() + >>> db = server.create('python-tests') + >>> idx = db.index() + >>> list(idx) #doctest: +SKIP + [{'ddoc': None, + 'def': {'fields': [{'_id': 'asc'}]}, + 'name': '_all_docs', + 'type': 'special'}] + >>> del server['python-tests'] + + :return: iterator yielding `dict`'s describing each index + """ + return iter(self._list()['indexes']) diff --git a/couchdb/http.py b/couchdb/http.py index dd22319b..d201f3e0 100644 --- a/couchdb/http.py +++ b/couchdb/http.py @@ -38,8 +38,8 @@ from couchdb import util __all__ = ['HTTPError', 'PreconditionFailed', 'ResourceNotFound', - 'ResourceConflict', 'ServerError', 'Unauthorized', 'RedirectLimit', - 'Session', 'Resource'] + 'ResourceConflict', 'ServerError', 'Unauthorized', 'Forbidden', + 'RedirectLimit', 'Session', 'Resource'] __docformat__ = 'restructuredtext en' @@ -114,6 +114,12 @@ class Unauthorized(HTTPError): """ +class Forbidden(HTTPError): + """Exception raised when the request requires an authorisation that the + current user does not have. + """ + + class RedirectLimit(Exception): """Exception raised when a request is redirected more often than allowed by the maximum number of redirections. @@ -160,24 +166,39 @@ def _release_conn(self): def close(self): while not self.resp.isclosed(): - self.resp.read(CHUNK_SIZE) + chunk = self.resp.read(CHUNK_SIZE) + if not chunk: + self.resp.close() if self.conn: self._release_conn() def iterchunks(self): assert self.chunked + buffer = [] while True: + if self.resp.isclosed(): break + chunksz = int(self.resp.fp.readline().strip(), 16) if not chunksz: self.resp.fp.read(2) #crlf self.resp.close() self._release_conn() break + chunk = self.resp.fp.read(chunksz) - for ln in chunk.splitlines(): - yield ln + for ln in chunk.splitlines(True): + + end = ln == b'\n' and not buffer # end of response + if not ln or end: + break + + buffer.append(ln) + if ln.endswith(b'\n'): + yield b''.join(buffer) + buffer = [] + self.resp.fp.read(2) #crlf @@ -347,6 +368,14 @@ def _try_request(): if num_redirects > self.max_redirects: raise RedirectLimit('Redirection limit exceeded') location = resp.getheader('location') + + # in case of relative location: add scheme and host to the location + location_split = util.urlsplit(location) + + if not location_split[0]: + orig_url_split = util.urlsplit(url) + location = util.urlunsplit(orig_url_split[:2] + location_split[2:]) + if status == 301: self.perm_redirects[url] = location elif status == 303: @@ -388,6 +417,8 @@ def _try_request(): error = '' if status == 401: raise Unauthorized(error) + elif status == 403: + raise Forbidden(error) elif status == 404: raise ResourceNotFound(error) elif status == 409: @@ -550,7 +581,10 @@ def _request(self, method, path=None, body=None, headers=None, **params): all_headers = self.headers.copy() all_headers.update(headers or {}) if path is not None: - url = urljoin(self.url, path, **params) + if isinstance(path, list): + url = urljoin(self.url, *path, **params) + else: + url = urljoin(self.url, path, **params) else: url = urljoin(self.url, **params) return self.session.request(method, url, body=body, @@ -569,7 +603,7 @@ def _request_json(self, method, path=None, body=None, headers=None, **params): def extract_credentials(url): """Extract authentication (user name and password) credentials from the given URL. - + >>> extract_credentials('http://localhost:5984/_config/') ('http://localhost:5984/_config/', None) >>> extract_credentials('http://joe:secret@localhost:5984/_config/') @@ -597,8 +631,8 @@ def basic_auth(credentials): >>> basic_auth(()) """ if credentials: - token = b64encode(('%s:%s' % credentials).encode('latin1')) - return ('Basic %s' % token.strip().decode('latin1')).encode('ascii') + token = b64encode(('%s:%s' % credentials).encode('utf-8')) + return ('Basic %s' % token.strip().decode('utf-8')).encode('ascii') def quote(string, safe=''): diff --git a/couchdb/loader.py b/couchdb/loader.py new file mode 100755 index 00000000..e593da3c --- /dev/null +++ b/couchdb/loader.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Load design documents from the filesystem into a dict. +Subset of couchdbkit/couchapp functionality. + +Description +----------- + +Convert a target directory into an object (dict). + +Each filename (without extension) or subdirectory name is a key in this object. + +For files, the utf-8-decoded contents are the value, except for .json files +which are first decoded as json. + +Subdirectories are converted into objects using the same procedure and then +added to the parent object. + +Typically used for design documents. This directory tree:: + + . + ├── filters + │ └── forms_only.js + ├── _id + ├── language + ├── lib + │ └── validate.js + └── views + ├── view_a + │ └── map.js + ├── view_b + │ └── map.js + └── view_c + └── map.js + +Becomes this object:: + + { + "views": { + "view_a": { + "map": "function(doc) { ... }" + }, + "view_b": { + "map": "function(doc) { ... }" + }, + "view_c": { + "map": "function(doc) { ... }" + } + }, + "_id": "_design/name_of_design_document", + "filters": { + "forms_only": "function(doc, req) { ... }" + }, + "language": "javascript", + "lib": { + "validate": "// A library for validations ..." + } + } + +""" + +from __future__ import unicode_literals, absolute_import + +import os.path +import pprint +import codecs +import json + +class DuplicateKeyError(ValueError): + pass + +def load_design_doc(directory, strip=False, predicate=lambda x: True): + """ + Load a design document from the filesystem. + + strip: remove leading and trailing whitespace from file contents, + like couchdbkit. + + predicate: function that is passed the full path to each file or directory. + Each entry is only added to the document if predicate returns True. + Can be used to ignore backup files etc. + """ + objects = {} + + if not os.path.isdir(directory): + raise OSError("No directory: '{0}'".format(directory)) + + for (dirpath, dirnames, filenames) in os.walk(directory, topdown=False): + key = os.path.split(dirpath)[-1] + ob = {} + objects[dirpath] = (key, ob) + + for name in filenames: + fkey = os.path.splitext(name)[0] + fullname = os.path.join(dirpath, name) + if not predicate(fullname): continue + if fkey in ob: + raise DuplicateKeyError("file '{0}' clobbers key '{1}'" + .format(fullname, fkey)) + with codecs.open(fullname, 'r', 'utf-8') as f: + contents = f.read() + if name.endswith('.json'): + contents = json.loads(contents) + elif strip: + contents = contents.strip() + ob[fkey] = contents + + for name in dirnames: + if name == '_attachments': + raise NotImplementedError("_attachments are not supported") + fullpath = os.path.join(dirpath, name) + if not predicate(fullpath): continue + subkey, subthing = objects[fullpath] + if subkey in ob: + raise DuplicateKeyError("directory '{0}' clobbers key '{1}'" + .format(fullpath,subkey)) + ob[subkey] = subthing + + return ob + + +def main(): + import sys + try: + directory = sys.argv[1] + except IndexError: + sys.stderr.write("Usage:\n\t{0} [directory]\n".format(sys.argv[0])) + sys.exit(1) + obj = load_design_doc(directory) + sys.stdout.write(json.dumps(obj, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/couchdb/mapping.py b/couchdb/mapping.py index bedddc13..626aaedc 100644 --- a/couchdb/mapping.py +++ b/couchdb/mapping.py @@ -406,6 +406,8 @@ def _wrap_row(cls, row): return cls.wrap(doc) data = row['value'] data['_id'] = row['id'] + if 'rev' in data: # When data is client.Document + data['_rev'] = data['rev'] return cls.wrap(data) @@ -472,12 +474,16 @@ def _to_json(self, value): class DateTimeField(Field): """Mapping field for storing date/time values. - + >>> field = DateTimeField() >>> field._to_python('2007-04-01T15:30:00Z') datetime.datetime(2007, 4, 1, 15, 30) - >>> field._to_json(datetime(2007, 4, 1, 15, 30, 0, 9876)) + >>> field._to_python('2007-04-01T15:30:00.009876Z') + datetime.datetime(2007, 4, 1, 15, 30, 0, 9876) + >>> field._to_json(datetime(2007, 4, 1, 15, 30, 0)) '2007-04-01T15:30:00Z' + >>> field._to_json(datetime(2007, 4, 1, 15, 30, 0, 9876)) + '2007-04-01T15:30:00.009876Z' >>> field._to_json(date(2007, 4, 1)) '2007-04-01T00:00:00Z' """ @@ -485,9 +491,15 @@ class DateTimeField(Field): def _to_python(self, value): if isinstance(value, util.strbase): try: - value = value.split('.', 1)[0] # strip out microseconds - value = value.rstrip('Z') # remove timezone separator - value = datetime(*strptime(value, '%Y-%m-%dT%H:%M:%S')[:6]) + split_value = value.split('.') # strip out microseconds + if len(split_value) == 1: # No microseconds provided + value = split_value[0] + value = value.rstrip('Z') #remove timezone separator + value = datetime.strptime(value, '%Y-%m-%dT%H:%M:%S') + else: + value = value.rstrip('Z') + value = datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%f') + except ValueError: raise ValueError('Invalid ISO date/time %r' % value) return value @@ -497,12 +509,12 @@ def _to_json(self, value): value = datetime.utcfromtimestamp(timegm(value)) elif not isinstance(value, datetime): value = datetime.combine(value, time(0)) - return value.replace(microsecond=0).isoformat() + 'Z' + return value.isoformat() + 'Z' class TimeField(Field): """Mapping field for storing times. - + >>> field = TimeField() >>> field._to_python('15:30:00') datetime.time(15, 30) diff --git a/couchdb/tests/__main__.py b/couchdb/tests/__main__.py index c3d9d9a0..ebc8d257 100644 --- a/couchdb/tests/__main__.py +++ b/couchdb/tests/__main__.py @@ -9,7 +9,8 @@ import unittest from couchdb.tests import client, couch_tests, design, couchhttp, \ - multipart, mapping, view, package, tools + multipart, mapping, view, package, tools, \ + loader def suite(): @@ -23,6 +24,7 @@ def suite(): suite.addTest(couch_tests.suite()) suite.addTest(package.suite()) suite.addTest(tools.suite()) + suite.addTest(loader.suite()) return suite diff --git a/couchdb/tests/_loader/_id b/couchdb/tests/_loader/_id new file mode 100644 index 00000000..4e85db7d --- /dev/null +++ b/couchdb/tests/_loader/_id @@ -0,0 +1 @@ +_design/loader diff --git a/couchdb/tests/_loader/filters.xml b/couchdb/tests/_loader/filters.xml new file mode 100644 index 00000000..22694cbd --- /dev/null +++ b/couchdb/tests/_loader/filters.xml @@ -0,0 +1 @@ +
Assert clobber of 'filters' directory
\ No newline at end of file diff --git a/couchdb/tests/_loader/filters/filter.js b/couchdb/tests/_loader/filters/filter.js new file mode 100644 index 00000000..d10461f1 --- /dev/null +++ b/couchdb/tests/_loader/filters/filter.js @@ -0,0 +1 @@ +function(doc, req) { return true; } diff --git a/couchdb/tests/_loader/language b/couchdb/tests/_loader/language new file mode 100644 index 00000000..a059bb79 --- /dev/null +++ b/couchdb/tests/_loader/language @@ -0,0 +1 @@ +javascript diff --git a/couchdb/tests/_loader/language.xml b/couchdb/tests/_loader/language.xml new file mode 100644 index 00000000..737b39dd --- /dev/null +++ b/couchdb/tests/_loader/language.xml @@ -0,0 +1 @@ +Assert clobber of 'language' (without an extension)
diff --git a/couchdb/tests/_loader/views/a/map.js b/couchdb/tests/_loader/views/a/map.js new file mode 100644 index 00000000..2c7a5409 --- /dev/null +++ b/couchdb/tests/_loader/views/a/map.js @@ -0,0 +1,3 @@ +function(doc) { + emit(doc.property_to_index); +} diff --git a/couchdb/tests/client.py b/couchdb/tests/client.py index 3792d941..c21f3940 100644 --- a/couchdb/tests/client.py +++ b/couchdb/tests/client.py @@ -136,6 +136,20 @@ def test_basic_auth(self): dbname = 'couchdb-python/test_basic_auth' self.assertRaises(http.Unauthorized, server.create, dbname) + def test_user_management(self): + url = client.DEFAULT_BASE_URL + if not isinstance(url, util.utype): + url = url.decode('utf-8') + + server = client.Server(url) + try: + server.add_user('foo', 'secret', roles=['hero']) + token = server.login('foo', 'secret') + self.assertTrue(server.verify_token(token)) + self.assertTrue(server.logout(token)) + finally: + server.remove_user('foo') + class DatabaseTestCase(testutil.TempDatabaseMixin, unittest.TestCase): @@ -360,6 +374,89 @@ def test_query_multi_get(self): for idx, i in enumerate(range(1, 6, 2)): self.assertEqual(i, res[idx].key) + def test_find(self): + if self.server.version_info()[0] < 2: + return + + docs = [ + dict(type='Person', name='John Doe'), + dict(type='Person', name='Mary Jane'), + dict(type='City', name='Gotham City') + ] + self.db.update(docs) + # the sort needs an index over `name`, the selector selects by `type` + idx = self.db.index() + idx['foo', 'bar'] = [{'type': 'asc'}, {'name': 'asc'}] + + res = list(self.db.find( + { + 'selector': { + 'type': 'Person' + }, + 'fields': ['name'], + # we need to specify the complete index here + 'sort': [{'type': 'asc'}, {'name': 'asc'}] + } + )) + self.assertEqual(2, len(res)) + expect = ['John Doe', 'Mary Jane'] + for i, doc in enumerate(res): + self.assertEqual(set(['name']), doc.keys()) + self.assertEqual(expect[i], doc['name']) + + def test_explain(self): + if self.server.version_info()[0] < 2: + return + mango = {'selector': {'type': 'Person'}, 'fields': ['name']} + res = self.db.explain(mango) + + self.assertEqual(['name'], res['fields']) + self.assertEqual({'type': {'$eq': 'Person'}}, res['selector']) + self.assertEqual(0, res['skip']) + self.assertEqual(self.db.name, res['dbname']) + + def test_index(self): + if self.server.version_info()[0] < 2: + return + + res = list(self.db.index()) + self.assertEqual(1, len(res)) + self.assertEqual({'ddoc': None, 'def': {'fields': [{'_id': 'asc'}]}, + 'name': '_all_docs', 'type': 'special'}, + res[0]) + + def test_add_index(self): + if self.server.version_info()[0] < 2: + return + + idx = self.db.index() + idx['foo', 'bar'] = [{'type': 'asc'}] + idxs = list(idx) + + self.assertEqual(2, len(idxs)) + for i in idxs: + if i['ddoc'] is not None: # special `_all_docs` index + self.assertEqual({'ddoc': '_design/foo', + 'def': {'fields': [{'type': 'asc'}]}, + 'name': 'bar', + 'type': 'json'}, + i) + return + self.failed() + + def test_remove_index(self): + if self.server.version_info()[0] < 2: + return + + idx = self.db.index() + idx['foo', 'bar'] = [{'type': 'asc'}] + res = list(idx) + self.assertEqual(2, len(res)) + del idx['foo', 'bar'] + + res = list(idx) + self.assertEqual(1, len(res)) + def test_bulk_update_conflict(self): docs = [ dict(type='Person', name='John Doe'), @@ -486,6 +583,28 @@ def test_changes_conn_usable(self): # in a good state from the previous request. self.assertTrue(self.db.info()['doc_count'] == 0) + def test_changes_conn_usable_selector(self): + if self.server.version_info()[0] < 2: + return + # Consume a changes feed to get a used connection in the pool. + list(self.db.changes(feed='continuous', + filter='_selector', + timeout=0, + _selector={'selector': {}})) + # Try using the connection again to make sure the connection was left + # in a good state from the previous request. + self.assertTrue(self.db.info()['doc_count'] == 0) + + def test_changes_usable_selector(self): + if self.server.version_info()[0] < 2: + return + # Consume a changes feed to get a used connection in the pool. + list(self.db.changes(filter='_selector', + _selector={'selector': {}})) + # Try using the connection again to make sure the connection was left + # in a good state from the previous request. + self.assertTrue(self.db.info()['doc_count'] == 0) + def test_changes_heartbeat(self): def wakeup(): time.sleep(.3) diff --git a/couchdb/tests/couch_tests.py b/couchdb/tests/couch_tests.py index f9e642eb..872690df 100644 --- a/couchdb/tests/couch_tests.py +++ b/couchdb/tests/couch_tests.py @@ -191,6 +191,31 @@ def test_utf8_encoding(self): for idx, row in enumerate(self.db.query(query)): self.assertEqual(texts[idx], row.key) + def test_update_with_unsafe_doc_ids(self): + doc_id = 'sanitise/the/doc/id/plz/' + design_doc = 'test_slashes_in_doc_ids' + handler_name = 'test' + func = """ + function(doc, req) { + doc.test = 'passed'; + return [doc, 'ok']; + } + """ + + # Stick an update handler in + self.db['_design/%s' % design_doc] = { + 'updates': {handler_name: func} + } + # And a test doc + self.db[doc_id] = {'test': 'failed'} + + response = self.db.update_doc( + '%s/%s' % (design_doc, handler_name), + docid=doc_id + ) + + self.assertEqual(self.db[doc_id]['test'], 'passed') + def test_design_docs(self): for i in range(50): self.db[str(i)] = {'integer': i, 'string': str(i)} diff --git a/couchdb/tests/couchhttp.py b/couchdb/tests/couchhttp.py index 41c42f4e..8646cf58 100644 --- a/couchdb/tests/couchhttp.py +++ b/couchdb/tests/couchhttp.py @@ -64,11 +64,11 @@ def close(self): def isclosed(self): return len(self.fp.getvalue()) == self.fp.tell() - data = b'foobarbaz' + data = b'foobarbaz\n' data = b'\n'.join([hex(len(data))[2:].encode('utf-8'), data]) response = http.ResponseBody(TestHttpResp(util.StringIO(data)), None, None, None) - self.assertEqual(list(response.iterchunks()), [b'foobarbaz']) + self.assertEqual(list(response.iterchunks()), [b'foobarbaz\n']) self.assertEqual(list(response.iterchunks()), []) diff --git a/couchdb/tests/loader.py b/couchdb/tests/loader.py new file mode 100644 index 00000000..f6eb7f16 --- /dev/null +++ b/couchdb/tests/loader.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2016 Daniel Holth +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. + +import unittest +import os.path + +from couchdb import loader +from couchdb.tests import testutil + +expected = { + '_id': u'_design/loader', + 'filters': {'filter': u'function(doc, req) { return true; }'}, + 'language': u'javascript', + 'views': {'a': {'map': u'function(doc) {\n emit(doc.property_to_index);\n}'}}} + +class LoaderTestCase(unittest.TestCase): + + directory = os.path.join(os.path.dirname(__file__), '_loader') + + def test_loader(self): + doc = loader.load_design_doc(self.directory, + strip=True, + predicate=lambda x: \ + not x.endswith('.xml')) + self.assertEqual(doc, expected) + + def test_bad_directory(self): + def bad_directory(): + doc = loader.load_design_doc('directory_does_not_exist') + + self.assertRaises(OSError, bad_directory) + + def test_clobber_1(self): + def clobber(): + doc = loader.load_design_doc(self.directory, + strip=True, + predicate=lambda x: \ + not x.endswith('filters.xml')) + + self.assertRaises(loader.DuplicateKeyError, clobber) + + def test_clobber_2(self): + def clobber(): + doc = loader.load_design_doc(self.directory, + strip=True, + predicate=lambda x: \ + not x.endswith('language.xml')) + + self.assertRaises(loader.DuplicateKeyError, clobber) + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(LoaderTestCase)) + return suite + + +if __name__ == '__main__': + unittest.main(defaultTest='suite') diff --git a/couchdb/tests/mapping.py b/couchdb/tests/mapping.py index 94387cb3..7f49b690 100644 --- a/couchdb/tests/mapping.py +++ b/couchdb/tests/mapping.py @@ -11,6 +11,7 @@ from couchdb import design, mapping from couchdb.tests import testutil +from datetime import datetime class DocumentTestCase(testutil.TempDatabaseMixin, unittest.TestCase): @@ -83,6 +84,15 @@ def test_old_datetime(self): dt = mapping.DateTimeField() assert dt._to_python('1880-01-01T00:00:00Z') + def test_datetime_with_microseconds(self): + dt = mapping.DateTimeField() + assert dt._to_python('2016-06-09T21:21:49.739248Z') + + def test_datetime_to_json(self): + dt = mapping.DateTimeField() + d = datetime.now() + assert dt._to_json(d) + def test_get_has_default(self): doc = mapping.Document() doc.get('foo') @@ -242,6 +252,12 @@ def test_view(self): include_docs=True) self.assertEqual(type(results.rows[0]), self.Item) + def test_wrapped_view(self): + self.Item().store(self.db) + results = self.db.view('_all_docs', wrapper=self.Item._wrap_row) + doc = results.rows[0] + self.db.delete(doc) + def test_query(self): self.Item().store(self.db) results = self.Item.query(self.db, all_map_func, None) diff --git a/couchdb/tests/package.py b/couchdb/tests/package.py index 862bdca8..33f12f81 100644 --- a/couchdb/tests/package.py +++ b/couchdb/tests/package.py @@ -11,7 +11,7 @@ def test_exports(self): 'Server', 'Database', 'Document', # couchdb.http 'HTTPError', 'PreconditionFailed', 'ResourceNotFound', - 'ResourceConflict', 'ServerError', 'Unauthorized', + 'ResourceConflict', 'ServerError', 'Unauthorized', 'Forbidden', 'Resource', 'Session' ]) exported = set(e for e in dir(couchdb) if not e.startswith('_')) diff --git a/couchdb/tests/view.py b/couchdb/tests/view.py index 79a18dd9..99f7f8d1 100644 --- a/couchdb/tests/view.py +++ b/couchdb/tests/view.py @@ -53,7 +53,7 @@ def test_map_doc_with_logging(self): view.run(input=input, output=output) self.assertEqual(output.getvalue(), b'true\n' - b'{"log": "running"}\n' + b'["log", "running"]\n' b'[[[null, {"foo": "bar"}]]]\n') def test_map_doc_with_logging_json(self): @@ -64,7 +64,7 @@ def test_map_doc_with_logging_json(self): view.run(input=input, output=output) self.assertEqual(output.getvalue(), b'true\n' - b'{"log": "[1, 2, 3]"}\n' + b'["log", "[1, 2, 3]"]\n' b'[[[null, {"foo": "bar"}]]]\n') def test_reduce(self): @@ -82,7 +82,7 @@ def test_reduce_with_logging(self): output = StringIO() view.run(input=input, output=output) self.assertEqual(output.getvalue(), - b'{"log": "Summing (1, 2, 3)"}\n' + b'["log", "Summing (1, 2, 3)"]\n' b'[true, [6]]\n') def test_rereduce(self): diff --git a/couchdb/tools/dump.py b/couchdb/tools/dump.py index c69a30d0..9ed78d2e 100755 --- a/couchdb/tools/dump.py +++ b/couchdb/tools/dump.py @@ -11,6 +11,7 @@ file. """ +from __future__ import print_function from base64 import b64decode from optparse import OptionParser import sys @@ -25,7 +26,7 @@ def dump_docs(envelope, db, docs): for doc in docs: - print >> sys.stderr, 'Dumping document %r' % doc.id + print('Dumping document %r' % doc.id, file=sys.stderr) attachments = doc.pop('_attachments', {}) jsondoc = json.encode(doc) @@ -57,7 +58,10 @@ def dump_docs(envelope, db, docs): }) def dump_db(dburl, username=None, password=None, boundary=None, - output=sys.stdout, bulk_size=BULK_SIZE): + output=None, bulk_size=BULK_SIZE): + + if output is None: + output = sys.stdout if sys.version_info[0] < 3 else sys.stdout.buffer db = Database(dburl) if username is not None and password is not None: diff --git a/couchdb/tools/load.py b/couchdb/tools/load.py index 95f04196..0d5b7866 100755 --- a/couchdb/tools/load.py +++ b/couchdb/tools/load.py @@ -11,6 +11,7 @@ file. """ +from __future__ import print_function from base64 import b64encode from optparse import OptionParser import sys @@ -36,7 +37,7 @@ def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False): doc['_attachments'] = {} else: doc['_attachments'][headers['content-id']] = { - 'data': b64encode(payload), + 'data': b64encode(payload).decode('ascii'), 'content_type': headers['content-type'], 'length': len(payload) } @@ -45,13 +46,13 @@ def load_db(fileobj, dburl, username=None, password=None, ignore_errors=False): doc = json.decode(payload) del doc['_rev'] - print>>sys.stderr, 'Loading document %r' % docid + print('Loading document %r' % docid, file=sys.stderr) try: db[docid] = doc except Exception as e: if not ignore_errors: raise - print>>sys.stderr, 'Error: %s' % e + print('Error: %s' % e, file=sys.stderr) def main(): diff --git a/couchdb/tools/replicate.py b/couchdb/tools/replicate.py index f527ffa9..a60fcd67 100755 --- a/couchdb/tools/replicate.py +++ b/couchdb/tools/replicate.py @@ -40,7 +40,7 @@ def findpath(parser, s): cut = None for i in range(0, len(parts) + 1): try: - data = res.get_json(parts[:i])[2] + data = res.get_json('/'.join(parts[:i]))[2] except Exception: data = None if data and 'couchdb' in data: diff --git a/couchdb/view.py b/couchdb/view.py index 0bb7e315..1412996b 100755 --- a/couchdb/view.py +++ b/couchdb/view.py @@ -24,13 +24,15 @@ log = logging.getLogger('couchdb.view') -def run(input=sys.stdin, output=sys.stdout): +def run(input=sys.stdin, output=None): r"""CouchDB view function handler implementation for Python. :param input: the readable file-like object to read input from :param output: the writable file-like object to write output to """ functions = [] + if output is None: + output = sys.stdout if sys.version_info[0] < 3 else sys.stdout.buffer def _writejson(obj): obj = json.encode(obj) @@ -43,7 +45,7 @@ def _writejson(obj): def _log(message): if not isinstance(message, util.strbase): message = json.encode(message) - _writejson({'log': message}) + _writejson(['log', message]) def reset(config=None): del functions[:] @@ -55,15 +57,15 @@ def add_fun(string): try: util.pyexec(string, {'log': _log}, globals_) except Exception as e: - return {'error': { - 'id': 'map_compilation_error', - 'reason': e.args[0] - }} - err = {'error': { - 'id': 'map_compilation_error', - 'reason': 'string must eval to a function ' + return ['error', + 'map_compilation_error', + e.args[0] + ] + err = ['error', + 'map_compilation_error', + 'string must eval to a function ' '(ex: "def(doc): return 1")' - }} + ] if len(globals_) != 1: return err function = list(globals_.values())[0] @@ -93,15 +95,15 @@ def reduce(*cmd, **kwargs): except Exception as e: log.error('runtime error in reduce function: %s', e, exc_info=True) - return {'error': { - 'id': 'reduce_compilation_error', - 'reason': e.args[0] - }} - err = {'error': { - 'id': 'reduce_compilation_error', - 'reason': 'string must eval to a function ' + return ['error', + 'reduce_compilation_error', + e.args[0] + ] + err = ['error', + 'reduce_compilation_error', + 'string must eval to a function ' '(ex: "def(keys, values): return 1")' - }} + ] if len(globals_) != 1: return err function = list(globals_.values())[0] diff --git a/doc/conf.py b/doc/conf.py index dd548cc3..fc57dd40 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -93,7 +93,7 @@ # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. -html_theme = 'default' +html_theme = 'alabaster' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/doc/getting-started.rst b/doc/getting-started.rst index bac3e518..e08b704e 100644 --- a/doc/getting-started.rst +++ b/doc/getting-started.rst @@ -14,6 +14,10 @@ running elsewhere, set it up like this: >>> couch = couchdb.Server('http://example.com:5984/') +You can also pass authentication credentials and/or use SSL: + + >>> couch = couchdb.Server('https://username:password@host:port/') + You can create a new database from Python, or use an existing database: >>> db = couch.create('test') # newly created diff --git a/setup.py b/setup.py index 5bc3b8cd..9fdcbd65 100755 --- a/setup.py +++ b/setup.py @@ -16,11 +16,6 @@ has_setuptools = False -requirements = [] -if sys.version_info < (2, 6): - requirements += ['simplejson'] - - # Build setuptools-specific options (if installed). if not has_setuptools: print("WARNING: setuptools/distribute not available. Console scripts will not be installed.") @@ -33,9 +28,10 @@ 'couchdb-dump = couchdb.tools.dump:main', 'couchdb-load = couchdb.tools.load:main', 'couchdb-replicate = couchdb.tools.replicate:main', + 'couchdb-load-design-doc = couchdb.loader:main', ], }, - 'install_requires': requirements, + 'install_requires': [], 'test_suite': 'couchdb.tests.__main__.suite', 'zip_safe': True, } @@ -43,7 +39,7 @@ setup( name = 'CouchDB', - version = '1.0.1', + version = '1.2.1', description = 'Python library for working with CouchDB', long_description = \ """This is a Python library for CouchDB. It provides a convenient high level @@ -55,15 +51,15 @@ license = 'BSD', url = 'https://github.com/djc/couchdb-python/', classifiers = [ - 'Development Status :: 4 - Beta', + 'Development Status :: 6 - Mature', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Topic :: Database :: Front-Ends', 'Topic :: Software Development :: Libraries :: Python Modules', ], diff --git a/tox.ini b/tox.ini index 33406f6b..53929648 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py33, py34, py33-json, py34-json +envlist = py27, py34, py34-json, py35-json [testenv] deps = simplejson @@ -7,15 +7,10 @@ commands = python --version {envbindir}/python -m couchdb.tests -[testenv:py26] -commands = - python --version - {envbindir}/python -m couchdb.tests.__main__ - -[testenv:py33-json] -basepython = python3.3 -deps = - [testenv:py34-json] basepython = python3.4 deps = + +[testenv:py35-json] +basepython = python3.5 +deps =