diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 31650e6..32b18c7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -7,24 +7,27 @@ on: [push, pull_request] jobs: test: strategy: + fail-fast: false # To see all versions that fail. matrix: - os: [ubuntu, windows, macos] - python: [3.5, 3.6, 3.7, 3.8, 3.9] - + os: ["ubuntu", "windows", "macos"] + python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] # https://devguide.python.org/versions/#versions + exclude: + - os: "macos" + python: "3.7" # Prevent "The version '3.7' with architecture 'arm64' was not found for macOS 14.4.1" runs-on: ${{ matrix.os }}-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} + cache: 'pip' - name: Prepare environment run: | python -m pip install --upgrade pip - pip install virtualenv - run: make depend - run: make test diff --git a/.gitignore b/.gitignore index c994ccb..9daffe1 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,5 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST -**/__pycache__ \ No newline at end of file +**/__pycache__ +env/ \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29df67d..f73fca2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,18 +2,24 @@ stages: - build - test +# First, build the source and package it as a python wheel build: + tags: [x64, docker, linux] image: python:latest stage: build script: - - pip3 install virtualenv # a function to download a library from another Gitlab repos CI artifacts - - 'download_library(){ curl --create-dirs -o objectbox/lib/$1 -H "PRIVATE-TOKEN: $CI_API_TOKEN" $2; }' - - 'download_library x86_64/libobjectbox.so "${OBXLIB_URL_Linux64}"' - - 'download_library x86_64/libobjectbox.dylib "${OBXLIB_URL_Mac64}"' - - 'download_library armv7l/libobjectbox.so "${OBXLIB_URL_LinuxARMv7hf}"' - - 'download_library armv6l/libobjectbox.so "${OBXLIB_URL_LinuxARMv6hf}"' - - 'download_library AMD64/objectbox.dll "${OBXLIB_URL_Win64}"' + # URLs stored as CI variables, see https://docs.gitlab.com/ee/api/job_artifacts.html#download-a-single-artifact-file-from-specific-tag-or-branch + # FIXME Need to update URLs and extract from archives. + #- 'download_library(){ curl --create-dirs -o objectbox/lib/$1 -H "PRIVATE-TOKEN: $CI_API_TOKEN" $2; }' + #- 'download_library x86_64/libobjectbox.so "${OBXLIB_URL_Linux64}"' + #- 'download_library x86_64/libobjectbox.dylib "${OBXLIB_URL_Mac64}"' + #- 'download_library armv7l/libobjectbox.so "${OBXLIB_URL_LinuxARMv7hf}"' + #- 'download_library armv6l/libobjectbox.so "${OBXLIB_URL_LinuxARMv6hf}"' + #- 'download_library AMD64/objectbox.dll "${OBXLIB_URL_Win64}"' + - python -m pip install --upgrade pip + # Using released C library + - make depend - make test - make build artifacts: @@ -21,47 +27,34 @@ build: paths: - dist/*.whl +# Next, test the packaged wheel built by "build" .test: stage: test script: - pip3 install --user pytest - - rm -r objectbox - - pip3 install --user --force-reinstall dist/*.whl - - python3 -m pytest + - rm -r objectbox # todo this is ugly; let's copy required files in a sub-folder instead? + - pip3 install --user --force-reinstall dist/*.whl # Artifacts from the previous stage (downloaded by default) + - ${PYTHON} -m pytest + variables: + PYTHON: "python3" -.test:linux:x64: +test:linux:x64: extends: .test tags: [x64, docker, linux] - -test:linux:x64:3.4: - extends: .test:linux:x64 - image: python:3.4 - -test:linux:x64:3.5: - extends: .test:linux:x64 - image: python:3.5 - -test:linux:x64:3.6: - extends: .test:linux:x64 - image: python:3.6 - -test:linux:x64:3.7: - extends: .test:linux:x64 - image: python:3.7 - -test:linux:x64:3.8: - extends: .test:linux:x64 - image: python:3.8-rc - -test:linux:armv6hf: - extends: .test - tags: [armv6hf, docker, linux] - image: balenalib/raspberry-pi-python:3.7-stretch + image: python:$PYTHON_VERSION + parallel: + matrix: + # Note: Docker images will have an arbitrary minor version due to "if-not-present" pull policy. + # If this becomes a problem, we could e.g. specify a minor version explicitly. + - PYTHON_VERSION: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] test:linux:armv7hf: + extends: .test + tags: [armv7hf, shell, linux, python3] + +test:linux:aarch64: extends: .test - tags: [armv7hf, docker, linux] - image: python:3.7 + tags: [aarch64, shell, linux, python3] test:mac:x64: extends: .test @@ -69,4 +62,6 @@ test:mac:x64: test:windows:x64: extends: .test - tags: [windows, x64, shell, python3] \ No newline at end of file + tags: [windows, x64, shell, python3] + variables: + PYTHON: "python.exe" diff --git a/.gitlab/issue_templates/release.md b/.gitlab/issue_templates/release.md new file mode 100644 index 0000000..201e97d --- /dev/null +++ b/.gitlab/issue_templates/release.md @@ -0,0 +1,14 @@ +**Check-list** + +- [ ] Update version in `objectbox/__init__.py` +- [ ] Check/update dependencies: + - [ ] `requirements.txt`: test and increase max. supported versions + - [ ] Update the C library version in `download-c-lib.py` and `objectbox/c.py` +- [ ] Check GitLab CI passes on main branch +- [ ] Update `README.md`: ensure all info is up-to-date. +- [ ] Commit and push to GitHub +- [ ] Clean, run tests and build: `make all` +- [ ] Publish to PyPI: `make publish` + - For this, you will need our login data for https://pypi.org/account/login - it can be found in 1pass +- [ ] Create a GitHub release +- [ ] Announce in GitHub issues, create release announcement/blog post. diff --git a/.gitlab/merge_request_templates/Default.md b/.gitlab/merge_request_templates/Default.md new file mode 100644 index 0000000..7da5b06 --- /dev/null +++ b/.gitlab/merge_request_templates/Default.md @@ -0,0 +1,24 @@ +## What does this MR do? + +Addresses: #X+ + +## Author's checklist + +- [ ] The MR fully addresses the requirements of the associated task. +- [ ] I did a self-review of the changes and did not spot any issues. Among others, this includes: + * I added unit tests for new/changed behavior; all test pass. + * My code conforms to our coding standards and guidelines. + * My changes are prepared in a way that makes the review straightforward for the reviewer. + +## Review checklist + +- [ ] I reviewed all changes line-by-line and addressed relevant issues +- [ ] The requirements of the associated task are fully met +- [ ] I can confirm that: + * CI passes + * Coverage percentages do not decrease + * New code conforms to standards and guidelines + * If applicable, additional checks were done for special code changes (e.g. core performance, binary size, OSS licenses) + +/assign me + diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e55bbed --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,25 @@ +ObjectBox Python ChangeLog +========================== + +4.0.0 (2024-05-28) +------------------ + +* ObjectBox now supports vector search ("vector database") to enable efficient similarity searches. + This is particularly useful for AI/ML/RAG applications, e.g. image, audio, or text similarity. + Other use cases include sematic search or recommendation engines. + See https://docs.objectbox.io/ann-vector-search for details. +* The definition of entities (aka the data model) is now greatly simplified + * Type-specific property classes, e.g. `name: String`, `count: Int64`, `score: Float32` + * Automatic ID/UID and model management (i.e. add/remove/rename of entities and properties) + * Automatic discovery of @Entity classes +* Queries: property-based conditions, e.g. `box.query(City.name.starts_with("Be"))` +* Queries: logical operators, e.g. `box.query(City.name == "Berlin" | City.name == "Munich")` +* Convenient "Store" API (deprecates ObjectBox and Builder API) +* New examples added, illustrating an VectorSearch and AI/RAG application +* Stable flat public API provided by single top-level module objectbox +* Dependency flatbuffers: Updated to 24.3.50 +* Adjusting the version number to match the core version (4.0); we will be aligning on major versions from now on. + +Older Versions +-------------- +Please check https://github.com/objectbox/objectbox-python/releases for details. \ No newline at end of file diff --git a/Makefile b/Makefile index 2a4ffa3..579e6f4 100644 --- a/Makefile +++ b/Makefile @@ -63,5 +63,10 @@ clean: ## Clean build artifacts rm -rf *.egg-info publish: ## Publish the package built by `make build` - set -e ; \ - ${PYTHON} -m twine upload --verbose dist/objectbox*.whl \ No newline at end of file + set -e + @echo "****************************************************************" + @echo ">>> Please enter the API token when asked for a password. <<<" + @echo ">>> The API token starts with the prefix 'pypi-'. <<<" + @echo ">>> See https://pypi.org/help/#apitoken for details. <<<" + @echo "****************************************************************" + ${PYTHON} -m twine upload -u "__token__" --verbose dist/objectbox*.whl diff --git a/README.md b/README.md index 30c3319..2fce0fc 100644 --- a/README.md +++ b/README.md @@ -1,149 +1,109 @@ -ObjectBox Python API -==================== - -[ObjectBox](https://objectbox.io) is a superfast database for objects, now also available for Python with a simple CRUD API. - -* Python version: 3.4+ -* Platforms supported: - * Linux x86-64 (64-bit) - * Linux ARMv6hf (e.g. Raspberry PI Zero) - * Linux ARMv7hf (e.g. Raspberry PI 3) - * MacOS x86-64 (64-bit) - * MacOS arm64 (Apple silicon) - * Windows x86-64 (64-bit) - -Getting started ---------------- - -First of all, install the latest version: - -```bash -pip install --upgrade objectbox -``` - -To start using ObjectBox as a storage for your data, you need to define your model first. -The model consists of Python classes annotated with `@Entity` decorator. - -### Model IDs and UIDs - -Each Entity has to have an ID (unique among entities). -Properties need an ID as well (unique inside one Entity). -Both Entities and Properties must also have an UID, which is a globally unique identifier. - -For other ObjectBox supported languages, the binding takes care of assigning these IDs/UIDs but this feature is not yet implemented for Python. -To learn more, see [ObjectBox Java documentation](https://docs.objectbox.io/advanced/meta-model-ids-and-uids) - -#### model.py +ObjectBox Python +================ +[ObjectBox](https://objectbox.io) Python is a lightweight yet powerful on-device object and vector database. +Store Python objects and vectors directly with an easy-to-use CRUD API while enjoying exceptional speed and efficiency. +And because it's an embedded database, there's no setup required. + +Its advanced vector search empowers on-device AI applications including RAG, generative AI, and similarity searches. + +The ObjectBox database delivers high-performance on commodity hardware - locally, on-device. +On top, as an offline-first solution, ObjectBox makes sure your app reliably works offline as well as online +(via [Sync](https://objectbox.io/sync/)). + +_Table of Contents_ + +- [Feature Highlights](#feature-highlights) +- [Code Example (CRUD - Create, Read, Update, Delete)](#code-example-crud-create-read-update-delete) +- [Getting Started](#getting-started) +- [Alpha Notes](#alpha-notes) +- [Help wanted](#help-wanted) +- [Feedback](#feedback) +- [License](#license) + +Feature Highlights +------------------ + +🏁 **On-device vector database** - for AI apps that work any place.\ +🏁 **High performance** - superfast response rates enabling real-time applications.\ +🪂 **ACID compliant** - Atomic, Consistent, Isolated, Durable.\ +🌱 **Scalable** - grows with your app, handling millions of objects with ease.\ +💚 **Sustainable** - frugal on CPU, Memory and battery / power use, reducing CO2 emissions.\ +💐 **[Queries](https://docs.objectbox.io/queries)** - filter data as needed, even across relations.\ +💻 **Multiplatform** - Get native speed on your favorite platforms.\ +* Linux x86-64 (64-bit) +* Linux ARMv6hf (e.g. Raspberry PI Zero) +* Linux ARMv7hf (e.g. Raspberry PI 3) +* Linux ARMv8 (e.g. Raspberry PI 4, 5, etc.) +* MacOS x86-64 and arm64 (Intel 64-bit and Apple Silicon) +* Windows x86-64 (64-bit) + +Code Example: CRUD (Create, Read, Update, Delete) +------------------------------------------------- + +What does using ObjectBox in Python look like? ```python -from objectbox.model import * +from objectbox import Entity, Id, Store, String -@Entity(id=1, uid=1) +@Entity() class Person: - id = Id(id=1, uid=1001) - name = Property(str, id=2, uid=1002) - is_enabled = Property(bool, id=3, uid=1003) - # int can be stored with 64 (default), 32, 16 or 8 bit precision. - int64 = Property(int, id=4, uid=1004) - int32 = Property(int, type=PropertyType.int, id=5, uid=1005) - int16 = Property(int, type=PropertyType.short, id=6, uid=1006) - int8 = Property(int, type=PropertyType.byte, id=7, uid=1007) - # float can be stored with 64 or 32 (default) bit precision. - float64 = Property(float, id=8, uid=1008) - float32 = Property(float, type=PropertyType.float, id=9, uid=1009) - byte_array = Property(bytes, id=10, uid=1010) - # Regular properties are not stored. - transient = "" -``` + id = Id + name = String -### Using ObjectBox +# The ObjectBox Store represents a database; keep it around... +store = Store() -To actually use the database, you launch (or "build") it with the model you've just defined. -Afterwards, you can reuse the instance (`ob` in the example below) and use it to access "Entity Boxes" which hold your objects. +# Get a box for the "Person" entity; a Box is the main interaction point with objects and the database. +box = store.box(Person) -#### program.py - -```python -import objectbox -# from mypackage.model import Person - -# Configure ObjectBox: should be done only once in the whole program and the "ob" variable should be kept around -model = objectbox.Model() -model.entity(Person, last_property_id=objectbox.model.IdUid(10, 1010)) -model.last_entity_id = objectbox.model.IdUid(1, 1) -ob = objectbox.Builder().model(model).directory("db").build() - -# Open the box of "Person" entity. This can be called many times but you can also pass the variable around -box = objectbox.Box(ob, Person) - -id = box.put(Person(name="Joe Green")) # Create +person = Person(name = "Joe Green") +id = box.put(person) # Create person = box.get(id) # Read person.name = "Joe Black" box.put(person) # Update box.remove(person) # Delete ``` -Additionally, see the [TaskList example app](https://github.com/objectbox/objectbox-python/tree/main/example). After checking out this repository to run the example: -``` -// Set up virtual environment, download ObjectBox libraries -make depend +Ready for more? Check the [example folder](https://github.com/objectbox/objectbox-python/tree/main/example). -// Activate virtual environment... -// ...on Linux -source .venv/bin/activate -// ...on Windows -.venv\Scripts\activate +Getting started +--------------- +Latest version: 4.0.0 (2024-05-28) -// Run the example -python3 -m example +To install or update the latest version of ObjectBox, run this: -// Once done, leave the virtual environment -deactivate +```bash +pip install --upgrade objectbox ``` +Now you are ready to use ObjectBox in your Python project. -For more information and code examples, see the tests folder. The docs for other languages may also help you understand the basics. - -* ObjectBox Java/Dart/Flutter - https://docs.objectbox.io -* ObjectBox Go - https://golang.objectbox.io -* ObjectBox Swift - https://swift.objectbox.io - -Some features -------------- +Head over to the **[ObjectBox documentation](https://docs.objectbox.io)** +and learn how to setup your first entity classes. -* automatic transactions (ACID compliant) -* bulk operations +### Examples -Coming in the future --------------------- - -The goodness you know from the other ObjectBox language-bindings, e.g., - -* model management (no need to manually set id/uid) -* automatic model migration (no schema upgrade scripts etc.) -* powerful queries -* relations (to-one, to-many) -* asynchronous operations -* secondary indexes +Do you prefer to dive right into working examples? +We have you covered in the [example](https://github.com/objectbox/objectbox-python/tree/main/example) folder. +It comes with a task list application and a vector search example using cities (CLI app and Jupyter notebook). +For AI developers , we provide an "ollama" example, which integrates a local LLM (via [ollama](https://ollama.com)) +with ObjectBox to manage and search embeddings effectively. Help wanted ----------- - -ObjectBox for Python is still in an early stage with limited feature set (compared to other languages). -To bring all these features to Python, we're asking the community to help out. PRs are more than welcome! +ObjectBox for Python is open to contributions. The ObjectBox team will try its best to guide you and answer questions. See [CONTRIBUTING.md](https://github.com/objectbox/objectbox-python/blob/main/CONTRIBUTING.md) to get started. Feedback -------- - -Also, please let us know your feedback by opening an issue: for example, if you experience errors or if you have ideas -for how to improve the API. Thanks! +We are looking for your feedback! +Please let us know what you think about ObjectBox for Python and how we can improve it. License ------- ```text -Copyright 2019-2021 ObjectBox Ltd. All rights reserved. +Copyright 2019-2024 ObjectBox Ltd. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/benchmark.py b/benchmark.py index 820b3db..47e3489 100644 --- a/benchmark.py +++ b/benchmark.py @@ -1,7 +1,8 @@ import objectbox import time +from objectbox.store import Store from tests.model import TestEntity -from tests.common import remove_test_dir, load_empty_test_objectbox +from tests.common import create_test_store class ObjectBoxPerf: @@ -10,8 +11,8 @@ class ObjectBoxPerf: """ def __init__(self): - self.ob = load_empty_test_objectbox() - self.box = objectbox.Box(self.ob, TestEntity) + self.store = create_test_store() + self.box = self.store.box(TestEntity) def remove_all(self): self.box.remove_all() @@ -110,10 +111,10 @@ def __progress_bar(text, value, endvalue, bar_length=20): if __name__ == "__main__": - remove_test_dir() + Store.remove_db_files("testdata") obPerf = ObjectBoxPerf() executor = PerfExecutor(obPerf) executor.run(count=10000, runs=20) - remove_test_dir() + Store.remove_db_files("testdata") diff --git a/download-c-lib.py b/download-c-lib.py index e1f9e91..58d711d 100644 --- a/download-c-lib.py +++ b/download-c-lib.py @@ -6,7 +6,7 @@ # Script used to download objectbox-c shared libraries for all supported platforms. Execute by running `make get-lib` # on first checkout of this repo and any time after changing the objectbox-c lib version. -version = "v0.14.0" # see objectbox/c.py required_version +version = "v4.0.0" # see objectbox/c.py required_version variant = 'objectbox' # or 'objectbox-sync' base_url = "https://github.com/objectbox/objectbox-c/releases/download/" @@ -49,6 +49,7 @@ def download(rel_path: str): # Download the file from `url`, save it in a temporary directory and get the path to it (e.g. '/tmp/tmpb48zma') source_url = url_for(rel_path); + print(f"URL {source_url}") tmp_file, headers = urllib.request.urlretrieve(source_url) # extract the file diff --git a/example/README.md b/example/README.md new file mode 100644 index 0000000..f747f9f --- /dev/null +++ b/example/README.md @@ -0,0 +1,96 @@ +# ObjectBox-Python Examples + +This directory contains a couple of examples that demonstrate capabilities of ObjectBox using the Python API. + +```shell +cd example # assuming you are in project root dir +python3 -m venv venv +source venv/bin/activate +pip install objectbox +``` + +The following examples are available from this directory: + +- `tasks`: CRUD Example (see below for details) +- `vectorsearch-cities`: VectorSearch Example (see below for details) +- `ollama`: LLM + VectorSearch Embeddings Script Example (See [ollama/README.md](./ollama/README.md) for details) + + +## Example: Tasks + +This is our classic Tasks application using a CLI. + +``` +cd tasks +$ python main.py + +Welcome to the ObjectBox tasks-list app example. Type help or ? for a list of commands. +> new buy oat +> new buy yeast +> new bake bread +> ls + ID Created Finished Text + 1 Mon Apr 22 11:02:27 2024 buy oat + 2 Mon Apr 22 11:02:30 2024 buy yeast + 3 Mon Apr 22 11:02:34 2024 bake bread +> done 1 +> done 2 +> ls +> ls + ID Created Finished Text + 1 Mon Apr 22 11:02:27 2024 Mon Apr 22 11:03:02 2024 buy oat + 2 Mon Apr 22 11:02:30 2024 Mon Apr 22 11:03:18 2024 buy yeast + 3 Mon Apr 22 11:02:34 2024 bake bread +> exit +``` + +## Example: Vector-Search with Cities + +We have two formats of this example available: + + * [Jupyter notebook](vectorsearch-cities-notebook/Vector-Search-City.ipynb) + * CLI application; see below for details + +This example application starts with a pre-defined set of capital cities and their geo coordinates. +It allows to search for nearest neighbors of a city (`city_neighbors`) or by coordinates (`neighbors`) as well as adding more locations (`add`). + +``` +cd vector-search-cities +$ python main.py + +Welcome to the ObjectBox vectorsearch-cities example. Type help or ? for a list of commands. +> ls +ID Name Latitude Longitude + 1 Abuja 9.08 7.40 + 2 Accra 5.60 -0.19 +[..] +212 Yerevan 40.19 44.52 +213 Zagreb 45.81 15.98 +> ls Ber +ID Name Latitude Longitude + 28 Berlin 52.52 13.40 + 29 Bern 46.95 7.45 +> city_neighbors Berlin +ID Name Latitude Longitude Score +147 Prague 50.08 14.44 7.04 + 49 Copenhagen 55.68 12.57 10.66 +200 Vienna 48.21 16.37 27.41 + 34 Bratislava 48.15 17.11 32.82 + 89 Ljubljana 46.06 14.51 42.98 +> neighbors 6,52.52,13.405 +ID Name Latitude Longitude Score + 28 Berlin 52.52 13.40 0.00 +147 Prague 50.08 14.44 7.04 + 49 Copenhagen 55.68 12.57 10.66 +200 Vienna 48.21 16.37 27.41 + 34 Bratislava 48.15 17.11 32.82 + 89 Ljubljana 46.06 14.51 42.98 + > add Area51, 37.23, -115.81 + > city_neighbors Area51 +ID Name Latitude Longitude Score +107 Mexico City 19.43 -99.13 594.86 + 27 Belmopan 17.25 -88.76 1130.92 + 64 Guatemala City 14.63 -90.51 1150.79 +164 San Salvador 13.69 -89.22 1261.12 + 67 Havana 23.11 -82.37 1317.73 +``` diff --git a/example/__init__.py b/example/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/example/model.py b/example/model.py deleted file mode 100644 index 88c7142..0000000 --- a/example/model.py +++ /dev/null @@ -1,18 +0,0 @@ -from objectbox.model import * - - -@Entity(id=1, uid=1) -class Task: - id = Id(id=1, uid=1001) - text = Property(str, id=2, uid=1002) - - # TODO property type DATE - date_created = Property(int, id=3, uid=1003) - date_finished = Property(int, id=4, uid=1004) - - -def get_objectbox_model(): - m = Model() - m.entity(Task, last_property_id=IdUid(4, 1004)) - m.last_entity_id = IdUid(1, 1) - return m diff --git a/example/ollama/README.md b/example/ollama/README.md new file mode 100644 index 0000000..01ec435 --- /dev/null +++ b/example/ollama/README.md @@ -0,0 +1,39 @@ +# Example: Using ObjectBox with Ollama + +based on https://ollama.com/blog/embedding-models + +## Setup + + 1. Install ollama. See instructions at https://ollama.com/download + + 2. Pull models + + ollama pull llama3 + ollama pull mxbai-embed-large + + 3. Change to example directory: + + cd example/ollama + + 3. Recommended: Create a new venv + + python3 -m venv .venv + source .venv/bin/activate + + 4. Install Python Bindings and ObjectBox: + + pip install ollama + pip install objectbox + + Or: + + pip install -r requirements.txt + + +## Run Example + +``` +$ python main.py + +Llamas are members of the camel family, which includes other large, even-toed ungulates such as camels, dromedaries, and Bactrian camels. Llamas are most closely related to alpacas, which are also native to South America and share many similarities in terms of their physical characteristics and behavior. Both llamas and alpacas belong to the family Camelidae, and are classified as ruminants due to their unique digestive system that allows them to break down cellulose in plant material. +``` diff --git a/example/ollama/main.py b/example/ollama/main.py new file mode 100644 index 0000000..402c0c4 --- /dev/null +++ b/example/ollama/main.py @@ -0,0 +1,67 @@ +# Example based on https://ollama.com/blog/embedding-models +# using objectbox as a vector store + +import ollama +from objectbox import * + +documents = [ + "Llamas are members of the camelid family meaning they're pretty closely related to vicuñas and camels", + "Llamas were first domesticated and used as pack animals 4,000 to 5,000 years ago in the Peruvian highlands", + "Llamas can grow as much as 6 feet tall though the average llama between 5 feet 6 inches and 5 feet 9 inches tall", + "Llamas weigh between 280 and 450 pounds and can carry 25 to 30 percent of their body weight", + "Llamas are vegetarians and have very efficient digestive systems", + "Llamas live to be about 20 years old, though some only live for 15 years and others live to be 30 years old", +] + +# Have fresh data for each start +Store.remove_db_files("objectbox") + +@Entity() +class DocumentEmbedding: + id = Id() + document = String() + embedding = Float32Vector(index=HnswIndex( + dimensions=1024, + distance_type=VectorDistanceType.COSINE + )) + +store = Store() +box = store.box(DocumentEmbedding) + +print("Documents to embed: ", len(documents)) + +# store each document in a vector embedding database +for i, d in enumerate(documents): + response = ollama.embeddings(model="mxbai-embed-large", prompt=d) + embedding = response["embedding"] + + box.put(DocumentEmbedding(document=d,embedding=embedding)) + print(f"Document {i + 1} embedded") + +# an example prompt +prompt = "What animals are llamas related to?" + +# generate an embedding for the prompt and retrieve the most relevant doc +response = ollama.embeddings( + prompt=prompt, + model="mxbai-embed-large" +) + +query = box.query( + DocumentEmbedding.embedding.nearest_neighbor(response["embedding"], 1) +).build() + +results = query.find_with_scores() +data = results[0][0].document + +print(f"Data most relevant to \"{prompt}\" : {data}") + +print("Generating the response now...") + +# generate a response combining the prompt and data we retrieved in step 2 +output = ollama.generate( + model="llama3", + prompt=f"Using this data: {data}. Respond to this prompt: {prompt}" +) + +print(output['response']) diff --git a/example/ollama/objectbox-model.json b/example/ollama/objectbox-model.json new file mode 100644 index 0000000..86a4d44 --- /dev/null +++ b/example/ollama/objectbox-model.json @@ -0,0 +1,35 @@ +{ + "_note1": "KEEP THIS FILE! Check it into a version control system (VCS) like git.", + "_note2": "ObjectBox manages crucial IDs for your object model. See docs for details.", + "_note3": "If you have VCS merge conflicts, you must resolve them according to ObjectBox docs.", + "modelVersionParserMinimum": 5, + "entities": [ + { + "id": "1:6961408191300375810", + "name": "DocumentEmbedding", + "lastPropertyId": "3:1544317969640594905", + "properties": [ + { + "id": "1:3876977054512990044", + "name": "id", + "type": 6, + "flags": 1 + }, + { + "id": "2:7160924628219144152", + "name": "document", + "type": 9 + }, + { + "id": "3:1544317969640594905", + "name": "embedding", + "type": 28, + "flags": 8, + "indexId": "1:3700549598877373380" + } + ] + } + ], + "lastEntityId": "1:6961408191300375810", + "lastIndexId": "1:3700549598877373380" +} \ No newline at end of file diff --git a/example/ollama/requirements.txt b/example/ollama/requirements.txt new file mode 100644 index 0000000..dad82fe --- /dev/null +++ b/example/ollama/requirements.txt @@ -0,0 +1,3 @@ +ollama +objectbox + diff --git a/example/__main__.py b/example/tasks/main.py similarity index 75% rename from example/__main__.py rename to example/tasks/main.py index 66d3ddd..0d2a349 100644 --- a/example/__main__.py +++ b/example/tasks/main.py @@ -1,23 +1,30 @@ from cmd import Cmd -import objectbox -import datetime -from example.model import * +from objectbox import * +import time + +@Entity() +class Task: + id = Id() + text = String() + + date_created = Date(py_type=int) + date_finished = Date(py_type=int) + # objectbox expects date timestamp in milliseconds since UNIX epoch def now_ms() -> int: - seconds: float = datetime.datetime.utcnow().timestamp() - return round(seconds * 1000) + return time.time_ns() / 1000000 def format_date(timestamp_ms: int) -> str: - return "" if timestamp_ms == 0 else str(datetime.datetime.fromtimestamp(timestamp_ms / 1000)) + return "" if timestamp_ms == 0 else time.ctime(timestamp_ms / 1000) class TasklistCmd(Cmd): prompt = "> " - _ob = objectbox.Builder().model(get_objectbox_model()).directory("tasklist-db").build() - _box = objectbox.Box(_ob, Task) + _store = Store(directory="tasklist-db") + _box = _store.box(Task) def do_ls(self, _): """list tasks""" diff --git a/example/tasks/objectbox-model.json b/example/tasks/objectbox-model.json new file mode 100644 index 0000000..8b57874 --- /dev/null +++ b/example/tasks/objectbox-model.json @@ -0,0 +1,41 @@ +{ + "_note1": "KEEP THIS FILE! Check it into a version control system (VCS) like git.", + "_note2": "ObjectBox manages crucial IDs for your object model. See docs for details.", + "_note3": "If you have VCS merge conflicts, you must resolve them according to ObjectBox docs.", + "modelVersionParserMinimum": 5, + "entities": [ + { + "id": "1:5152209066423907023", + "name": "Task", + "lastPropertyId": "4:7654179801863866748", + "properties": [ + { + "id": "1:705340479536670333", + "name": "id", + "type": 6, + "flags": 1 + }, + { + "id": "2:488411871914776672", + "name": "text", + "type": 9, + "flags": 0 + }, + { + "id": "3:3330502346908829733", + "name": "date_created", + "type": 10, + "flags": 0 + }, + { + "id": "4:7654179801863866748", + "name": "date_finished", + "type": 10, + "flags": 0 + } + ] + } + ], + "lastEntityId": "1:5152209066423907023", + "lastIndexId": "0:0" +} \ No newline at end of file diff --git a/example/vectorsearch-cities-notebook/Vector-Search-City.ipynb b/example/vectorsearch-cities-notebook/Vector-Search-City.ipynb new file mode 100644 index 0000000..8f98bb6 --- /dev/null +++ b/example/vectorsearch-cities-notebook/Vector-Search-City.ipynb @@ -0,0 +1,622 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "collapsed_sections": [ + "ZBNxYvPOs2Gq" + ] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# ObjectBox 4.0 Python City Example\n", + "\n", + "This Jupyter Notebook demonstrates ObjectBox 4.0 using City data.\n", + "\n", + "* Create the data model class, and the database\n", + "* Insert data\n", + "* Create a simple query\n", + "* Create a nearest neighbor vector search\n", + "\n", + "For more information on the 4.0 release and details on this first on-device vector database and its possibilities see our [blog post](https://objectbox.io/the-first-on-device-vector-database-objectbox-4-0).\n" + ], + "metadata": { + "id": "i9isobkArR76" + } + }, + { + "cell_type": "markdown", + "source": [ + "## First, install the latest version of ObjectBox" + ], + "metadata": { + "id": "9BvKIodbnIl4" + } + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "TrEe2BAP1FIn", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "cb6651af-f915-407e-989a-f44bdf7842a6", + "collapsed": true + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: objectbox in /usr/local/lib/python3.10/dist-packages (4.0.0a5)\n", + "Requirement already satisfied: flatbuffers==24.3.25 in /usr/local/lib/python3.10/dist-packages (from objectbox) (24.3.25)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from objectbox) (1.25.2)\n" + ] + } + ], + "source": [ + "!pip install --upgrade objectbox" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Create the ObjectBox data model\n", + "\n", + "We define a `City` class with its properties (name and location and an ID).\n", + "\n", + "*Note:* The `HnswIndex` is a special index for vectors that enables high-performance vector search." + ], + "metadata": { + "id": "m47QyjcsrbQO" + } + }, + { + "cell_type": "code", + "source": [ + "from objectbox import Entity, Float32Vector, HnswIndex, Id, Store, String\n", + "\n", + "@Entity()\n", + "class City:\n", + " id = Id()\n", + " name = String()\n", + " location = Float32Vector(index=HnswIndex(dimensions=2))" + ], + "metadata": { + "id": "MW_kVFC1iDpH" + }, + "execution_count": 2, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Create the ObjectBox `Store` and `Box`\n", + "\n", + "The `Store` is a database instance. From the `Strore` we prepare a `Box` to interact with `City` objects:" + ], + "metadata": { + "id": "5tpAvePQrkBO" + } + }, + { + "cell_type": "code", + "source": [ + "store = Store()\n", + "box = store.box(City)" + ], + "metadata": { + "id": "IKp3F4pikWCQ" + }, + "execution_count": 3, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## A list of capital cities (no database interaction yet)" + ], + "metadata": { + "id": "ZBNxYvPOs2Gq" + } + }, + { + "cell_type": "code", + "source": [ + "def city(name, lat, lon):\n", + " return City(name=name, location=[lat,lon])\n", + "\n", + "cities = [\n", + " city(\"Abuja\", 9.0765, 7.3986),\n", + " city(\"Accra\", 5.6037, -0.1870),\n", + " city(\"Addis Ababa\", 9.0084, 38.7813),\n", + " city(\"Algiers\", 36.7529, 3.0420),\n", + " city(\"Amman\", 31.9632, 35.9306),\n", + " city(\"Amsterdam\", 52.3667, 4.8945),\n", + " city(\"Ankara\", 39.9334, 32.8597),\n", + " city(\"Antananarivo\", -18.8792, 47.5079),\n", + " city(\"Apia\", -13.8330, -171.7667),\n", + " city(\"Ashgabat\", 37.9601, 58.3261),\n", + " city(\"Asmara\", 15.3229, 38.9251),\n", + " city(\"Astana\", 51.1796, 71.4475),\n", + " city(\"Asunción\", -25.2637, -57.5759),\n", + " city(\"Athens\", 37.9795, 23.7162),\n", + " city(\"Avarua\", -21.2079, -159.7750),\n", + " city(\"Baghdad\", 33.3152, 44.3661),\n", + " city(\"Baku\", 40.4093, 49.8671),\n", + " city(\"Bamako\", 12.6530, -7.9864),\n", + " city(\"Bandar Seri Begawan\", 4.9031, 114.9398),\n", + " city(\"Bangkok\", 13.7563, 100.5018),\n", + " city(\"Bangui\", 4.3947, 18.5582),\n", + " city(\"Banjul\", 13.4549, -16.5790),\n", + " city(\"Basseterre\", 17.3026, -62.7177),\n", + " city(\"Beijing\", 39.9042, 116.4074),\n", + " city(\"Beirut\", 33.8889, 35.4944),\n", + " city(\"Belgrade\", 44.7866, 20.4489),\n", + " city(\"Belmopan\", 17.2510, -88.7590),\n", + " city(\"Berlin\", 52.5200, 13.4050),\n", + " city(\"Bern\", 46.9480, 7.4474),\n", + " city(\"Bishkek\", 42.8746, 74.5698),\n", + " city(\"Bissau\", 11.8636, -15.5842),\n", + " city(\"Bogotá\", 4.7109, -74.0721),\n", + " city(\"Brasília\", -15.8267, -47.9218),\n", + " city(\"Bratislava\", 48.1486, 17.1077),\n", + " city(\"Brazzaville\", -4.2634, 15.2429),\n", + " city(\"Bridgetown\", 13.1132, -59.5988),\n", + " city(\"Brussels\", 50.8503, 4.3517),\n", + " city(\"Bucharest\", 44.4268, 26.1025),\n", + " city(\"Budapest\", 47.4979, 19.0402),\n", + " city(\"Buenos Aires\", -34.6037, -58.3816),\n", + " city(\"Bujumbura\", -3.3818, 29.3622),\n", + " city(\"Cairo\", 30.0444, 31.2357),\n", + " city(\"Canberra\", -35.2809, 149.1300),\n", + " city(\"Caracas\", 10.4806, -66.9036),\n", + " city(\"Castries\", 14.0101, -60.9874),\n", + " city(\"Chisinau\", 47.0105, 28.8638),\n", + " city(\"Colombo\", 6.9271, 79.8612),\n", + " city(\"Conakry\", 9.6412, -13.5784),\n", + " city(\"Copenhagen\", 55.6761, 12.5683),\n", + " city(\"Dakar\", 14.7167, -17.4677),\n", + " city(\"Damascus\", 33.5131, 36.2919),\n", + " city(\"Dhaka\", 23.8103, 90.4125),\n", + " city(\"Dili\", -8.5569, 125.5603),\n", + " city(\"Djibouti\", 11.5890, 43.1456),\n", + " city(\"Dodoma\", -6.1748, 35.7469),\n", + " city(\"Doha\", 25.2854, 51.5310),\n", + " city(\"Dublin\", 53.3498, -6.2603),\n", + " city(\"Dushanbe\", 38.5868, 68.7841),\n", + " city(\"Freetown\", 8.4840, -13.2299),\n", + " city(\"Funafuti\", -8.5210, 179.1962),\n", + " city(\"Gaborone\", -24.6282, 25.9231),\n", + " city(\"Georgetown\", 6.8013, -58.1550),\n", + " city(\"Gibraltar\", 36.1408, -5.3536),\n", + " city(\"Guatemala City\", 14.6349, -90.5069),\n", + " city(\"Hanoi\", 21.0278, 105.8342),\n", + " city(\"Harare\", -17.8252, 31.0335),\n", + " city(\"Havana\", 23.1136, -82.3666),\n", + " city(\"Helsinki\", 60.1699, 24.9384),\n", + " city(\"Honiara\", -9.4376, 159.9720),\n", + " city(\"Islamabad\", 33.6844, 73.0479),\n", + " city(\"Jakarta\", -6.2088, 106.8456),\n", + " city(\"Juba\", 4.8594, 31.5713),\n", + " city(\"Kabul\", 34.5553, 69.2075),\n", + " city(\"Kampala\", 0.3476, 32.5825),\n", + " city(\"Kathmandu\", 27.7172, 85.3240),\n", + " city(\"Khartoum\", 15.5007, 32.5599),\n", + " city(\"Kiev\", 50.4501, 30.5234),\n", + " city(\"Kigali\", -1.9441, 30.0619),\n", + " city(\"Kingston\", 17.9710, -76.7924),\n", + " city(\"Kingstown\", 13.1467, -61.2121),\n", + " city(\"Kinshasa\", -4.4419, 15.2663),\n", + " city(\"Kuala Lumpur\", 3.1390, 101.6869),\n", + " city(\"Kuwait City\", 29.3759, 47.9774),\n", + " city(\"La Paz\", -16.4897, -68.1193),\n", + " city(\"Libreville\", 0.4162, 9.4673),\n", + " city(\"Lilongwe\", -13.9626, 33.7741),\n", + " city(\"Lima\", -12.0464, -77.0428),\n", + " city(\"Lisbon\", 38.7223, -9.1393),\n", + " city(\"Ljubljana\", 46.0569, 14.5058),\n", + " city(\"Lomé\", 6.1319, 1.2228),\n", + " city(\"London\", 51.5072, -0.1276),\n", + " city(\"Luanda\", -8.8399, 13.2894),\n", + " city(\"Lusaka\", -15.3875, 28.3228),\n", + " city(\"Luxembourg City\", 49.6116, 6.1319),\n", + " city(\"Madrid\", 40.4168, -3.7038),\n", + " city(\"Majuro\", 7.1164, 171.1859),\n", + " city(\"Malabo\", 3.7508, 8.7839),\n", + " city(\"Male\", 4.1755, 73.5093),\n", + " city(\"Mamoudzou\", -12.7871, 45.2750),\n", + " city(\"Managua\", 12.1364, -86.2514),\n", + " city(\"Manama\", 26.2285, 50.5860),\n", + " city(\"Manila\", 14.5995, 120.9842),\n", + " city(\"Maputo\", -25.8918, 32.6051),\n", + " city(\"Maseru\", -29.2976, 27.4854),\n", + " city(\"Mbabane\", -26.3054, 31.1367),\n", + " city(\"Melekeok\", 7.4874, 134.6265),\n", + " city(\"Mexico City\", 19.4326, -99.1332),\n", + " city(\"Minsk\", 53.9045, 27.5615),\n", + " city(\"Mogadishu\", 2.0469, 45.3182),\n", + " city(\"Monaco\", 43.7325, 7.4189),\n", + " city(\"Monrovia\", 6.3005, -10.7974),\n", + " city(\"Montevideo\", -34.9011, -56.1645),\n", + " city(\"Moroni\", -11.7022, 43.2551),\n", + " city(\"Moscow\", 55.7558, 37.6173),\n", + " city(\"Muscat\", 23.5859, 58.4059),\n", + " city(\"Nairobi\", -1.2921, 36.8219),\n", + " city(\"Nassau\", 25.0478, -77.3554),\n", + " city(\"Naypyidaw\", 19.7633, 96.0785),\n", + " city(\"New Delhi\", 28.6139, 77.2090),\n", + " city(\"Ngerulmud\", 7.5004, 134.6249),\n", + " city(\"Niamey\", 13.5122, 2.1254),\n", + " city(\"Nicosia\", 35.1725, 33.365),\n", + " city(\"Nicosia Northern Cyprus\", 35.19, 33.363611),\n", + " city(\"Nouakchott\", 18.0735, -15.9582),\n", + " city(\"Nuku'alofa\", -21.1393, -175.2049),\n", + " city(\"Nuuk\", 64.1836, -51.7214),\n", + " city(\"Oranjestad\", 12.5092, -70.0086),\n", + " city(\"Oslo\", 59.9139, 10.7522),\n", + " city(\"Ottawa\", 45.4215, -75.6972),\n", + " city(\"Ouagadougou\", 12.3714, -1.5197),\n", + " city(\"Pago Pago\", -14.2794, -170.7004),\n", + " city(\"Palikir\", 6.9248, 158.1614),\n", + " city(\"Panama City\", 8.9824, -79.5199),\n", + " city(\"Papeete\", -17.5350, -149.5699),\n", + " city(\"Paramaribo\", 5.8520, -55.2038),\n", + " city(\"Paris\", 48.8566, 2.3522),\n", + " city(\"Philipsburg\", 18.0255, -63.0450),\n", + " city(\"Phnom Penh\", 11.5564, 104.9282),\n", + " city(\"Plymouth\", 16.7056, -62.2126),\n", + " city(\"Podgorica\", 42.4304, 19.2594),\n", + " city(\"Port Louis\", -20.1619, 57.4989),\n", + " city(\"Port Moresby\", -9.4438, 147.1803),\n", + " city(\"Port Vila\", -17.7416, 168.3213),\n", + " city(\"Port-au-Prince\", 18.5944, -72.3074),\n", + " city(\"Port of Spain\", 10.6596, -61.4789),\n", + " city(\"Porto-Novo\", 6.4968, 2.6283),\n", + " city(\"Prague\", 50.0755, 14.4378),\n", + " city(\"Praia\", 14.9195, -23.5087),\n", + " city(\"Pretoria\", -25.7463, 28.1876),\n", + " city(\"Pristina\", 42.6629, 21.1655),\n", + " city(\"Pyongyang\", 39.0392, 125.7625),\n", + " city(\"Quito\", -0.1807, -78.4678),\n", + " city(\"Rabat\", 33.9693, -6.9275),\n", + " city(\"Reykjavik\", 64.1466, -21.9426),\n", + " city(\"Riga\", 56.9496, 24.1052),\n", + " city(\"Riyadh\", 24.7136, 46.6753),\n", + " city(\"Road Town\", 18.4207, -64.6399),\n", + " city(\"Rome\", 41.9028, 12.4964),\n", + " city(\"Roseau\", 15.3092, -61.3794),\n", + " city(\"Saipan\", 15.1833, 145.7500),\n", + " city(\"San José\", 9.9281, -84.0907),\n", + " city(\"San Juan\", 18.4655, -66.1057),\n", + " city(\"San Marino\", 43.9424, 12.4578),\n", + " city(\"San Salvador\", 13.6929, -89.2182),\n", + " city(\"Sana'a\", 15.3694, 44.1910),\n", + " city(\"Santiago\", -33.4489, -70.6693),\n", + " city(\"Santo Domingo\", 18.4861, -69.9312),\n", + " city(\"Sarajevo\", 43.8564, 18.4131),\n", + " city(\"Seoul\", 37.5665, 126.9780),\n", + " city(\"Singapore\", 1.3521, 103.8198),\n", + " city(\"Skopje\", 41.9973, 21.4279),\n", + " city(\"Sofia\", 42.6975, 23.3241),\n", + " city(\"Sri Jayawardenepura Kotte\", 6.8928, 79.9277),\n", + " city(\"St. George's\", 12.0561, -61.7485),\n", + " city(\"St. Helier\", 49.1839, -2.1064),\n", + " city(\"St. John's\", 17.1171, -61.8456),\n", + " city(\"St. Peter Port\", 49.4599, -2.5352),\n", + " city(\"Stanley\", -51.7020, -57.8517),\n", + " city(\"Stockholm\", 59.3293, 18.0686),\n", + " city(\"Sucre\", -19.0421, -65.2559),\n", + " city(\"Sukhumi\", 43.0004, 41.0234),\n", + " city(\"Suva\", -18.1416, 178.4419),\n", + " city(\"Taipei\", 25.0330, 121.5654),\n", + " city(\"Tallinn\", 59.4370, 24.7536),\n", + " city(\"Tarawa\", 1.4170, 173.0000),\n", + " city(\"Tashkent\", 41.2995, 69.2401),\n", + " city(\"Tbilisi\", 41.7151, 44.8271),\n", + " city(\"Tegucigalpa\", 14.0818, -87.2068),\n", + " city(\"Tehran\", 35.6892, 51.3890),\n", + " city(\"Thimphu\", 27.4728, 89.6390),\n", + " city(\"Tirana\", 41.3275, 19.8187),\n", + " city(\"Tokyo\", 35.6762, 139.6503),\n", + " city(\"Tripoli\", 32.8867, 13.1910),\n", + " city(\"Tunis\", 36.8065, 10.1815),\n", + " city(\"Ulaanbaatar\", 47.8864, 106.9057),\n", + " city(\"Vaduz\", 47.1410, 9.5215),\n", + " city(\"Valletta\", 35.9042, 14.5189),\n", + " city(\"Vatican City\", 41.9029, 12.4534),\n", + " city(\"Victoria\", -4.6182, 55.4515),\n", + " city(\"Vienna\", 48.2082, 16.3738),\n", + " city(\"Vientiane\", 17.9757, 102.6331),\n", + " city(\"Vilnius\", 54.6872, 25.2797),\n", + " city(\"Warsaw\", 52.2297, 21.0122),\n", + " city(\"Washington, D.C.\", 38.9072, -77.0369),\n", + " city(\"Wellington\", -41.2865, 174.7762),\n", + " city(\"West Island\", -12.1880, 96.8292),\n", + " city(\"Willemstad\", 12.1091, -68.9319),\n", + " city(\"Windhoek\", -22.5749, 17.0805),\n", + " city(\"Yamoussoukro\", 6.8276, -5.2893),\n", + " city(\"Yaoundé\", 3.8480, 11.5021),\n", + " city(\"Yaren\", -0.5467, 166.9209),\n", + " city(\"Yerevan\", 40.1872, 44.5152),\n", + " city(\"Zagreb\", 45.8150, 15.9819)\n", + "]\n", + "\n" + ], + "metadata": { + "id": "71uGkBVIs4K4" + }, + "execution_count": 4, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Put `City` objects into the Database\n", + "\n", + "Now, that we have our list of City object, we can `put` them in the database. Note that we also call `remove_all()` to make this step repeatable:\n" + ], + "metadata": { + "id": "V0cnHX58rtTt" + } + }, + { + "cell_type": "code", + "source": [ + "box.remove_all() # Remove previous data (if any) to avoid duplicates\n", + "box.put(*cities)\n", + "print(\"Cities in the database:\", box.count())" + ], + "metadata": { + "id": "j97ubc-Rpuwr", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "0eaf962c-e249-443f-8d5a-70a6369bf664" + }, + "execution_count": 5, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Cities in the database: 213\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## A simple Query\n", + "\n", + "To demonstrate a simple query, let's list all Cities starting with \"Be\":" + ], + "metadata": { + "id": "ilqsxtyquYj_" + } + }, + { + "cell_type": "code", + "source": [ + "query = box.query(City.name.starts_with(\"Be\")).build()\n", + "results = query.find()\n", + "print(f\"Found {len(results)} objects\")\n", + "for city in results:\n", + " print(f\"{city.name:>10s} {city.location}\")\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "QPUFiOJruftx", + "outputId": "46843958-14a8-4a61-a3f7-6219493c15a1" + }, + "execution_count": 6, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Found 6 objects\n", + " Beijing [ 39.9042 116.4074]\n", + " Beirut [33.8889 35.4944]\n", + " Belgrade [44.7866 20.4489]\n", + " Belmopan [ 17.251 -88.759]\n", + " Berlin [52.52 13.405]\n", + " Bern [46.948 7.4474]\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "##Nearest-neighbor Vector Search\n", + "\n", + "OK, now let's perform a *nearest-neighbor* search! List the 15 nearest cities closest to a given query location:" + ], + "metadata": { + "id": "jSAXKDSrrz0e" + } + }, + { + "cell_type": "code", + "source": [ + "query_location = [51.0, 12.0] # Somewhere in Germany, south-west of Berlin\n", + "\n", + "query = box.query(City.location.nearest_neighbor(query_location, 15)).build()\n", + "results = query.find_with_scores()\n", + "\n", + "print(f\"Found {len(results)} objects\")\n", + "for i, result in enumerate(results):\n", + " capital, score = result\n", + " print(f\"{i + 1}. Capital: {capital.name}, Score: {score}\")\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Bvg1y9QrzDRy", + "outputId": "2ceeeae6-e4ee-406c-a316-0600ba6af7da" + }, + "execution_count": 7, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Found 15 objects\n", + "1. Capital: Berlin, Score: 4.284425735473633\n", + "2. Capital: Prague, Score: 6.79757022857666\n", + "3. Capital: Vaduz, Score: 21.034852981567383\n", + "4. Capital: Copenhagen, Score: 22.188892364501953\n", + "5. Capital: Vienna, Score: 26.924283981323242\n", + "6. Capital: Ljubljana, Score: 30.713272094726562\n", + "7. Capital: Bratislava, Score: 34.21907424926758\n", + "8. Capital: Luxembourg City, Score: 36.36225891113281\n", + "9. Capital: Bern, Score: 37.14485549926758\n", + "10. Capital: Zagreb, Score: 42.739768981933594\n", + "11. Capital: San Marino, Score: 50.01927185058594\n", + "12. Capital: Amsterdam, Score: 52.35599899291992\n", + "13. Capital: Brussels, Score: 58.51890563964844\n", + "14. Capital: Budapest, Score: 61.829124450683594\n", + "15. Capital: Monaco, Score: 73.80305480957031\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "##Visualize the results\n", + "\n", + "And finally, let's visualize an the search result using the matplotlib:" + ], + "metadata": { + "id": "wowqid_swoBp" + } + }, + { + "cell_type": "code", + "source": [ + "!pip install matplotlib" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "id": "re0O1-2lnHrO", + "outputId": "914e77f7-3e37-4238-bb25-87f1b8e5af87" + }, + "execution_count": 8, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (3.7.1)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.2.1)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (4.51.0)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.4.5)\n", + "Requirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.25.2)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (24.0)\n", + "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (9.4.0)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (3.1.2)\n", + "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (2.8.2)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "The search results (aka the nearest neighbors) are the blue dots and the search location is the red dot:" + ], + "metadata": { + "id": "OT0IdzEQnsmf" + } + }, + { + "cell_type": "code", + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "for result in results:\n", + " city = result[0]\n", + " lat = city.location[0]\n", + " lon = city.location[1]\n", + " plt.scatter(lon, lat, color='blue', s=50, alpha=0.8)\n", + " plt.annotate(city.name, (lon, lat), textcoords=\"offset points\",\n", + " xytext=(0, -10), ha='center', fontsize=8)\n", + "\n", + "plt.scatter(query_location[1], query_location[0], color='red', s=50, alpha=0.8)\n", + "\n", + "plt.xlabel('Longitude')\n", + "plt.ylabel('Latitude')\n", + "plt.title('Nearest Capitals')\n", + "\n", + "plt.show()\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 492 + }, + "id": "E8_3SqcCwsHV", + "outputId": "07bc8643-af10-4778-e0de-67949d795c37" + }, + "execution_count": 9, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjYAAAHHCAYAAACskBIUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABtZklEQVR4nO3deXzM1/7H8dfILhJrrCWxBiEJUVttRVEuRSwldkp1sbRa9OoVSmltbWlLtRUVWi2llOpVS1utXcZSqrhSaSitJYvIPr8/5pepEUtEkkkm7+fjMY8x55zv+X6+QfLJ+Z5zvgaTyWRCRERExA4UsXUAIiIiIjlFiY2IiIjYDSU2IiIiYjeU2IiIiIjdUGIjIiIidkOJjYiIiNgNJTYiIiJiN5TYiIiIiN1QYiMiIiJ2Q4mNiEgOiIyMxGAwEBYWlqP9+vj4MGTIkBztU8SeKbERKWDCwsIwGAy4uroSHR2dqb5NmzbUq1fPBpHlrISEBEJDQ9m5c+d9HXfx4kUmTJhA7dq1KVq0KO7u7gQFBTFjxgyuXbuWK7HeyebNmwkNDc3Tc4oUdo62DkBEsicpKYnZs2ezcOFCW4eSKxISEpg2bRpgTtayYv/+/XTu3Jn4+HgGDBhAUFAQAAcOHGD27Nn88MMP/Pe//82VeL29vblx4wZOTk6Wss2bN/Puu+8quRHJQ0psRAqowMBAli5dyuTJk6lYsaKtw+H69eu4u7vb7PzXrl2jR48eODg4EBERQe3ata3qZ86cydKlS3Pt/BmjaCJiW7oVJVJAvfLKK6SlpTF79uwstQ8PDycoKAg3NzdKlSrFk08+SVRUlFWbH3/8kd69e1OlShVcXFyoXLky48eP58aNG1bthgwZQrFixThz5gydO3fGw8ODkJAQANLT03nrrbfw8/PD1dWVcuXKMWrUKK5evWrVx4EDB+jYsSNlypTBzc2NqlWrMmzYMMA8X8XLywuAadOmYTAYMBgMdx35WLJkCdHR0cyfPz9TUgNQrlw5pkyZYvn81Vdf0aVLFypWrIiLiwvVq1fntddeIy0tzeq4jFt7Bw8epHnz5pZYFy9ebNXu1jk2Q4YM4d133wWwxG8wGCzt586dS/PmzSldujRubm4EBQWxZs2aO15fhpSUFKZNm0bNmjVxdXWldOnStGjRgq1bt97zWJHCQCM2IgVU1apVGTRoEEuXLmXSpEl3HbWZOXMmr776Kn369GHEiBH89ddfLFy4kFatWhEREUGJEiUA+OKLL0hISGD06NGULl2affv2sXDhQv744w+++OILqz5TU1Pp2LEjLVq0YO7cuRQtWhSAUaNGERYWxtChQxkzZgxnz55l0aJFRERE8NNPP+Hk5MSlS5fo0KEDXl5eTJo0iRIlShAZGcmXX34JgJeXF++//z6jR4+mR48e9OzZEwB/f/87XuOGDRtwc3OjV69eWfr6hYWFUaxYMV544QWKFSvG9u3b+c9//kNsbCxz5syxanv16lU6d+5Mnz596NevH59//jmjR4/G2dnZkozdatSoUZw/f56tW7eyYsWKTPVvv/023bp1IyQkhOTkZD777DN69+7N119/TZcuXe4Yd2hoKLNmzWLEiBE0btyY2NhYDhw4wKFDh3jssceydO0ids0kIgXKsmXLTIBp//79pjNnzpgcHR1NY8aMsdS3bt3a5OfnZ/kcGRlpcnBwMM2cOdOqn6NHj5ocHR2tyhMSEjKdb9asWSaDwWD6/fffLWWDBw82AaZJkyZZtf3xxx9NgGnlypVW5Vu2bLEqX7duneUa7uSvv/4yAaapU6fe5avxj5IlS5oCAgKy1NZkuv21jho1ylS0aFFTYmKipax169YmwDRv3jxLWVJSkikwMNBUtmxZU3JysslkMpnOnj1rAkzLli2ztHv22WdNd/o2e+v5k5OTTfXq1TO1bdvWqtzb29s0ePBgy+eAgABTly5dsnydIoWNbkWJFGDVqlVj4MCBfPDBB1y4cOG2bb788kvS09Pp06cPf//9t+VVvnx5atasyY4dOyxt3dzcLH++fv06f//9N82bN8dkMhEREZGp79GjR1t9/uKLLyhevDiPPfaY1bmCgoIoVqyY5VwZI0Rff/01KSkpD/plACA2NhYPD48st7/5WuPi4vj7779p2bIlCQkJ/Prrr1ZtHR0dGTVqlOWzs7Mzo0aN4tKlSxw8eDBb8d58/qtXrxITE0PLli05dOjQXY8rUaIEv/zyC6dOncrWeUXsnRIbkQJuypQppKam3nGuzalTpzCZTNSsWRMvLy+r14kTJ7h06ZKl7blz5xgyZAilSpWiWLFieHl50bp1awBiYmKs+nV0dOShhx7KdK6YmBjKli2b6Vzx8fGWc7Vu3Zrg4GCmTZtGmTJleOKJJ1i2bBlJSUnZ/jp4enoSFxeX5fa//PILPXr0oHjx4nh6euLl5cWAAQNue60VK1bMNDG6Vq1agHluTXZ8/fXXNG3aFFdXV0qVKmW5/XbruW81ffp0rl27Rq1atahfvz4vvfQSR44cyVYMIvZIc2xECrhq1aoxYMAAPvjgAyZNmpSpPj09HYPBwDfffIODg0Om+mLFigGQlpbGY489xpUrV5g4cSK1a9fG3d2d6OhohgwZQnp6utVxLi4uFCli/btReno6ZcuWZeXKlbeNNWNCsMFgYM2aNezZs4eNGzfy7bffMmzYMObNm8eePXssMd2P2rVrYzQaSU5OxtnZ+a5tr127RuvWrfH09GT69OlUr14dV1dXDh06xMSJEzNda0778ccf6datG61ateK9996jQoUKODk5sWzZMlatWnXXY1u1asWZM2f46quv+O9//8uHH37IggULWLx4MSNGjMjVuEUKAiU2InZgypQphIeH88Ybb2Sqq169OiaTiapVq1pGGW7n6NGj/PbbbyxfvpxBgwZZyu9ntU316tX57rvveOSRR6xutdxJ06ZNadq0KTNnzmTVqlWEhITw2WefMWLECKsVRFnRtWtXdu/ezdq1a+nXr99d2+7cuZPLly/z5Zdf0qpVK0v52bNnb9v+/PnzmZaz//bbb4B5Z+A7udM1rF27FldXV7799ltcXFws5cuWLbtr3BlKlSrF0KFDGTp0KPHx8bRq1YrQ0FAlNiLoVpSIXahevToDBgxgyZIl/Pnnn1Z1PXv2xMHBgWnTpmEymazqTCYTly9fBrCM5tzcxmQy8fbbb2c5jj59+pCWlsZrr72WqS41NdWy8+/Vq1czxRIYGAhguR2Vscoqq7sFP/3001SoUIEXX3zRknTc7NKlS8yYMQO4/bUmJyfz3nvv3bbv1NRUlixZYtV2yZIleHl5WTYBvJ2MROjWa3BwcMBgMFgtLY+MjGT9+vV3v0iw/H1lKFasGDVq1Hig23gi9kQjNiJ24t///jcrVqzg5MmT+Pn5WcqrV6/OjBkzmDx5MpGRkXTv3h0PDw/Onj3LunXrGDlypOURBNWrV2fChAlER0fj6enJ2rVrM+0/czetW7dm1KhRzJo1C6PRSIcOHXBycuLUqVN88cUXvP322/Tq1Yvly5fz3nvv0aNHD6pXr05cXBxLly7F09OTzp07A+bJtXXr1mX16tXUqlWLUqVKUa9evTs+LqJkyZKsW7eOzp07ExgYaLXz8KFDh/j0009p1qwZAM2bN6dkyZIMHjyYMWPGYDAYWLFiRaZkK0PFihV54403iIyMpFatWqxevRqj0cgHH3xgtdPwrTLOP2bMGDp27IiDgwNPPvkkXbp0Yf78+XTq1In+/ftz6dIl3n33XWrUqHHP+TJ169alTZs2BAUFUapUKQ4cOMCaNWt47rnn7v6XI1JY2Go5lohkz83LvW+VsQz75uXeGdauXWtq0aKFyd3d3eTu7m6qXbu26dlnnzWdPHnS0ub48eOm9u3bm4oVK2YqU6aM6amnnjIdPnw40zLmwYMHm9zd3e8Y4wcffGAKCgoyubm5mTw8PEz169c3vfzyy6bz58+bTCaT6dChQ6Z+/fqZqlSpYnJxcTGVLVvW9K9//ct04MABq35+/vlnU1BQkMnZ2TnLS7/Pnz9vGj9+vKlWrVomV1dXU9GiRU1BQUGmmTNnmmJiYiztfvrpJ1PTpk1Nbm5upooVK5pefvll07fffmsCTDt27LC0y1g+f+DAAVOzZs1Mrq6uJm9vb9OiRYusznu75d6pqamm559/3uTl5WUyGAxWS78/+ugjU82aNU0uLi6m2rVrm5YtW2aaOnVqpuXhty73njFjhqlx48amEiVKmNzc3Ey1a9c2zZw507LsXKSwM5hMd/gVRUREaNOmDX///TfHjh2zdSgikgWaYyMiIiJ2Q4mNiIiI2A0lNiIiImI3NMdGRERE7IZGbERERMRuKLERERERu2H3G/Slp6dz/vx5PDw87nuLdhEREbENk8lEXFwcFStWzPRcurux+8Tm/PnzVK5c2dZhiIiISDZERUXx0EMPZbm93Sc2Hh4egPkL4+npaeNoREREJCtiY2OpXLmy5ed4Vtl9YpNx+8nT01OJjYiISAFzv9NINHlYRERE7IZNE5vQ0FAMBoPVq3bt2lZtdu/eTdu2bXF3d8fT05NWrVpx48YNG0UsIiIi+ZnNb0X5+fnx3XffWT47Ov4T0u7du+nUqROTJ09m4cKFODo6cvjw4fuaHS0iIiKFh80TG0dHR8qXL3/buvHjxzNmzBgmTZpkKfP19c2r0ERERKSAsfnQx6lTp6hYsSLVqlUjJCSEc+fOAXDp0iX27t1L2bJlad68OeXKlaN169bs2rXrrv0lJSURGxtr9RIREZHCwaaJTZMmTQgLC2PLli28//77nD17lpYtWxIXF8f//vc/wDwP56mnnmLLli00bNiQdu3acerUqTv2OWvWLIoXL255aQ8bERGRwiNfPQTz2rVreHt7M3/+fOrUqcMjjzzC5MmTef311y1t/P396dKlC7NmzbptH0lJSSQlJVk+Z6yDj4mJ0XJvERGRAiI2NpbixYvf989vm8+xuVmJEiWoVasWp0+fpm3btgDUrVvXqk2dOnUst6tux8XFBRcXl1yNU0RERPInm8+xuVl8fDxnzpyhQoUK+Pj4ULFiRU6ePGnV5rfffsPb29tGEYpIYZWUBFeumN9FJP+y6YjNhAkT6Nq1K97e3pw/f56pU6fi4OBAv379MBgMvPTSS0ydOpWAgAACAwNZvnw5v/76K2vWrLFl2CJSiBiNEB4OmzZBSgo4OUGXLjBwIAQE2Do6EbmVTRObP/74g379+nH58mW8vLxo0aIFe/bswcvLC4Bx48aRmJjI+PHjuXLlCgEBAWzdupXq1avbMmwRKSTWrIHJkyEuDooWNSc1SUnmROerr2D2bAgOtnWUInKzfDV5ODdkd/KRiBRuRiP07g03bkC5cnDz42pMJrh4EdzczMmPRm5Ecl52f37nqzk2IiL5RXi4eaTm1qQGzJ/LlTPXh4fbJj4RuT0lNiIit0hKMs+pKVo0c1KTwWAw12/apAnFIvmJEhsRkVtcv/7PROG7cXKC5GRzexHJH5TYiIjcwt3dnLSkpNy9XUoKODub24tI/qDERkTkFi4u5iXdCQnmicK3YzKZ67t0MbcXkfxBiY2IyG0MGAAeHubVT7cmNxmrojw8zO1EJP9QYiMichuBgeZ9atzcIDravOtwXJz5PTraXD57tpZ6i+Q3+epZUSIi+UlwMNSo8c/Ow8nJ4OpqLh8wQEmNSH6kDfpEJN9LTU1l5syZfPrppzg6OuLo6Ejjxo158803KVGiRJ7EkJQE//lPKAkJ11i48K08OadIYWYXT/cWEbmd4cOHc+XKFXbv3k3JkiUxmUysWbOGK1eu5Fli4+Jivv2kPWtE8jclNiKSr50+fZovvviCc+fOUbJkSQAMBgO9e/cGYM6cOYSFhVGkSBH8/f157733KF68OKGhoRw9epSrV69y/vx5atasSVhYGKVLlyYlJYVXX32V7du3k5ycTK1atViyZAklS5ZkyJAhuLi4cPr0aaKioqhXrx6fffYZzs7OAFy4cIGuXbty5swZypcvz5o1ayhVqhRHjx5l9OjRJCQkkJiYSP/+/ZkyZQoAcXFxjBgxgsOHD+Pl5UXdunVJSkoiLCwMgLlz5/L555+TmppK2bJlWbJkCd7e3oSGhnLixAkSEhIynU9Ebk+Th0UkXzt06BA1a9akTJkymeq++eYbPv74Y3766SeOHj2Ku7s7kyZNstT/+OOPrFq1il9//ZXKlSszefJkwJwMubu7s2/fPoxGI/Xr17ckIQBGo5GNGzdy4sQJLl68yNq1ay11e/fuJSwsjOPHj1uSEAAfHx+2bdvGoUOHOHjwIGvXrmXPnj0ATJ8+HTc3N06cOMHmzZv5+eefLf2tWrWKkydPsnv3bg4dOkRISAjPPPPMPc8nIrenERsRKbC+++47+vbta7kdNXr0aMtIDkCXLl0oX748ACNHjqRnz54ArF+/npiYGEvCkpycjI+Pj+W4Hj16ULRoUQAaN27MmTNnLHWdOnWidOnSADRr1oyjR48CcOPGDZ555hmMRiNFihQhKioKo9FI06ZN2bZtGwsWLMBgMODh4UHfvn05ffq0JZb9+/cTFBQEQFpamtU13ul8InJ7SmxEJF9r2LAhp06d4vLly5Yf8HdiuNODnW6pN5lMLFy4kA4dOty2naurq+XPDg4OpKam3rPulVdeoUyZMkRERODo6EjPnj1JTEy8Z5wmk4nJkyczcuTI+45FRDLTrSgRyddq1KhBcHAww4cP59q1a4A5GVi7di3VqlXj888/JzY2FoAlS5ZYJSubN2/m4sWLAHz44Ye0b98egO7du7NgwQISEhIASEhI4JdffnmgOK9evcpDDz2Eo6MjJ0+eZOvWrZa6tm3bsnz5ckwmE/Hx8Xz++eeWuu7du7N48WKuXLkCQEpKChEREQ8Ui0hhphEbEcn3Pv74Y2bMmEGTJk1wdHQkPT2dVq1a8cYbb5CQkECzZs2sJg9naNmyJf379yc6OtoyeRhg4sSJJCUl0aRJE8voycSJE/Hz88t2jFOmTGHgwIEsX76c6tWr07ZtW0vdf/7zH4YPH06dOnUoU6YMAQEBlttnISEhXL58mUcffRQwL20fNmwYDRo0yHYsIoWZ9rEREbsUGhrKtWvXeOutt2wdCikpKaSlpeHq6sr169fp2LEjzz//PH379rV1aCL5lvaxERHJp65evcrjjz9OWloaiYmJPPHEE/Tp08fWYYnYJY3YiIiISL6T3Z/fmjwsIiIidkOJjYiIiNgNJTYiIiJiN5TYiIiIiN1QYiMiIiJ2Q4mNiIiI2A0lNiIiImI3lNiIiIiI3VBiIyIiInZDiY2IiIjYDSU2IiIiYjeU2IiIiIjdUGIjIiIidkOJjYiIiNgNJTYiIiJiN5TYiIiIiN1QYiMiIiJ2Q4mNiIiI2A0lNiIiImI3lNiIiIiI3VBiIyIiInZDiY2IiIjYDSU2IiIiYjeU2IiIiIjdUGIjIiIidkOJjYiIiNgNmyY2oaGhGAwGq1ft2rUztTOZTDz++OMYDAbWr1+f94GKiIhIgeBo6wD8/Pz47rvvLJ8dHTOH9NZbb2EwGPIyLBERESmAbJ7YODo6Ur58+TvWG41G5s2bx4EDB6hQoUIeRiYiIiIFjc3n2Jw6dYqKFStSrVo1QkJCOHfunKUuISGB/v378+677941+blZUlISsbGxVi8REREpHGya2DRp0oSwsDC2bNnC+++/z9mzZ2nZsiVxcXEAjB8/nubNm/PEE09kuc9Zs2ZRvHhxy6ty5cq5Fb6IiIjkMwaTyWSydRAZrl27hre3N/Pnz8fLy4sXX3yRiIgIihUrBoDBYGDdunV07979jn0kJSWRlJRk+RwbG0vlypWJiYnB09Mzty9BREREckBsbCzFixe/75/fNp9jc7MSJUpQq1YtTp8+zdGjRzlz5gwlSpSwahMcHEzLli3ZuXPnbftwcXHBxcUl94MVERGRfCdfJTbx8fGcOXOGgQMH0qdPH0aMGGFVX79+fRYsWEDXrl1tFKGIiIjkZzZNbCZMmEDXrl3x9vbm/PnzTJ06FQcHB/r164eXl9dtJwxXqVKFqlWr2iBaERERye9smtj88ccf9OvXj8uXL+Pl5UWLFi3Ys2cPXl5etgxLRERECiibJjafffbZfbXPR/OcRUREJB+y+T42IiIiIjlFiY2IiIjYDSU2IiIiYjeU2IiIiIjdUGIjIiIidkOJjYiIiNgNJTYiIiJiN5TYiIiIiN1QYiMiIiJ2Q4mNiIiI2A0lNiIiImI3lNiIiIiI3VBiIyIiInZDiY2IiIjYDSU2IiIiYjeU2IiIiIjdUGIjIiIidkOJjYiIiNgNJTYiIiJiN5TYiIiIiN1QYiMiIiJ2Q4mNiIiI2A0lNiIiImI3lNiIiIiI3VBiIyIiInZDiY2IiIjYDSU2IiIiYjeU2IiIiIjdUGIjIiIidkOJjYhIPpGUBFeumN9FJHscbR2AiEhhZzRCeDhs2gQpKeDkBF26wMCBEBBg6+hEChaN2IiI2NCaNdC7tzmxSUoCBwfze3g49OoFa9faOkKRgkUjNiIiNmI0wuTJcOMGVKoEBsM/dSVLwsWLMGkS1KihkRuRrNKIjQ3pfrpI4RYeDnFxUK6cdVID5s/lypnrw8NtE59IQaTExgaMRpgwAQIDoXFj8/uECXD4sI0DE5E8k5RknlNTtGjmpCaDwWCu37RJvwCJZJUSmzym++kiAnD9+j8The/GyQmSk83tReTeNMcmD+l+uohkcHc3Jy33GolJSQFXV3N7Ebk3jdjkId1PF5EMLi7mJd0JCWAy3b6NyWSu79LF3F5E7k2JTR7R/XQRudWAAeDhYR6tvTW5MZnM5R4e5nYikjVKbPKI7qeLyK0CA2H2bHBzg+ho8yrJuDjze3S0uXz2bN2aFrkfmmOTR3Q/XURuJzjYPK8uY+fh5GTz94DgYPNIjZIakfujEZs8ovvpIoWHj48Pvr6+BAYGUqdOHfr378/1uwzDBgTAnDkQEQH79pnf58yBJ57wwWg0AjBixAh27NiRR1cgUnApsclDup8uUnisXr0ao9HIL7/8QkxMDGFhYfc8xsUFSpUCJ6d00tPTreo+/PBDHn300VyKVsR+KLHJQ7qfLlL4JCcnk5CQQMmSJQGYO3cujRs3pmHDhnTq1Inff/8dgNDQUIKDg+nYsSP16tXjwoULVv20adOG9evXAzBkyBBGjRpFu3btqFWrFj179iQ5OTlPr0skv1Jik8eCg82b9A0caL6PnpZmfh840FweHGzrCEUkJ/Tt25fAwEDKly9PkSJF6NOnD6tWreLkyZPs3r2bQ4cOERISwjPPPGM5Zvfu3XzyySccP36cSpUq3bV/o9HIxo0bOXHiBBcvXmStdvcUAWyc2ISGhmIwGKxetWvXBuDKlSs8//zz+Pr64ubmRpUqVRgzZgwxMTG5EktcXBzFihVj+PDhOdpvWFgYv/76q1XZne6n32ukJj4+HsOd1oqLSL6ScSvq77//xsfHh4kTJ7J+/Xq+++47goKCCAwM5M033+TcuXOWYzp37ky5cuWy1H+PHj0oWrQoDg4ONG7cmDNnzuTWpYgUKDYfsfHz8+PChQuW165duwA4f/4858+fZ+7cuRw7doywsDC2bNmS44lHhtWrVxMUFMSXX35JfHx8jvV7u8QmQ8b99NtNFE5Pz3yPXUQKHkdHR4KDg9myZQsmk4nJkydjNBoxGo0cPXqUo0ePWtoWK1Ysy/26urpa/uzg4EBqamqOxi1SUNk8sXF0dKR8+fKWV5kyZQCoV68ea9eupWvXrlSvXp22bdsyc+ZMNm7cmCv/gT/66CMmTpxIq1atWL16NWBOStq3b0+/fv2oW7cuzZs35/jx4/To0YM6derQoUMHSxK0ceNG/P39CQwMpF69enz11Vd8+OGHHDhwgPHjxxMYGMjmzZuB+7vHvmTJEmrWrEmDBg1YsGCBVcwhISE0atQIf39/unTpwp9//glAZGQkJUqU4NVXX6Vhw4bUrFmTn376yRJHvXr1OHbsWI5/DUXk9rZv346vry/du3dn8eLFXLlyBYCUlBQiIiJsHJ2IfbF5YnPq1CkqVqxItWrVCAkJsRqWvVVMTAyenp44Ot55+52kpCRiY2OtXvdy/PhxoqKi6NixI8OHD+ejjz6y1O3fv5833niD48ePU716dbp27crixYs5ceIEzs7OLF++HIApU6awZMkSjEYjR44coXXr1owYMYJGjRqxYMECjEYjnTt3vq977FevXmXq1Kn88MMPREREcOPGDau433rrLQ4cOMCRI0do2bIloaGhVl+roKAgDh06xKRJk+jYsSPdunXDaDQyePBgpk2bds+vi4hkX8Ycm3r16nHixAnefvttQkJCGDJkCI8++igBAQEEBgayfft2W4cqYldsukFfkyZNCAsLw9fXlwsXLjBt2jRatmzJsWPH8PDwsGr7999/89prrzFy5Mi79jlr1qz7/qH90UcfMWjQIBwcHOjcuTOjRo3ixIkTADRr1owqVaoA0KhRI1JSUiz3wB9++GFOnToFQLt27Rg7diy9evWiQ4cOBAYG3vZc69evZ//+/QQFBQGQlpZmVX/zPfbt27fz+OOPU6FCBQBGjx7NrFmzLG1XrVrFihUrSExMJDEx0TLaBeZh6u7du1viLlasmGWpaOPGjVm5cuV9fY1EJOsiIyPvWDdmzBjGjBmTqfzmX0xu18/OnTstf7516fjcuXPvM0IR+2XTEZvHH3+c3r174+/vT8eOHdm8eTPXrl3j888/t2oXGxtLly5dqFu37m3/899s8uTJxMTEWF5RUVF3bZ+SksKKFStYvnw5Pj4+1KhRg4SEBMuoza33se90X3v+/PksW7aMokWLMnjwYN58883bnu9B7rHfPHF4165dvPPOO2zevJljx44xf/58EhMTLfUuN03cuVvcIiIi9sTmt6JuVqJECWrVqsXp06ctZXFxcXTq1AkPDw/WrVuH0z0etuTi4oKnp6fV6242bNhAtWrViI6OJjIyksjISPbs2cOKFStISUnJcuy//vorfn5+PPfcc4wePZo9e/YA4OnpabWS637usbdt25YtW7ZY5s4sXrzYUnf16lU8PDwoXbo0ycnJLFmyJMuxioiI2Kt8ldjEx8dz5swZy62X2NhYOnTogLOzMxs2bLAadcgpH330ESEhIVZlderUoVKlSsTFxWW5n1deeQU/Pz8aNGjAihUrLCNLI0eO5PXXX7dMHr6fe+z16tUjNDSUli1b0qBBA6tRmE6dOuHr64uvry8tW7a8460vERGRwsRgMt3pyUW5b8KECXTt2hVvb2/Onz/P1KlTMRqNHD9+HBcXFzp06EBCQgLr1q3D/aanQnp5eeHg4JClc8TGxlK8eHHLxGMRERHJ/7L789umk4f/+OMP+vXrx+XLl/Hy8qJFixbs2bMHLy8vdu7cyd69ewGoUaOG1XFnz57Fx8fHBhGLiIhIfmbTEZu8oBEbERGRgie7P7/z1RwbERERkQehxEZERETshhIbERERsRtKbERERMRuKLERERERu6HERkREROyGEhsRERGxG0psRERExG4osRERERG7ocRGRERE7IYSGxEREbEbSmxERETEbiixEREREbuhxEZERETshhIbERERsRtKbETEviUlwZUr5ncRsXuOtg5ARCRXGI0QHg6bNkFKCjg5QZcuMHAgBATYOjoRySUasRER+7NmDfTubU5skpLAwcH8Hh4OvXrB2rW2jlBEcolGbETEvhiNMHky3LgBlSqBwfBPXcmScPEiTJoENWpo5EbEDmnERkTsS3g4xMVBuXLWSQ2YP5crZ64PD7dNfCKSq5TYZJPmI4rkQ0lJ5jk1RYtmTmoyGAzm+k2b9B9YxA7pVtR90nxEkXzs+vV//mPejZMTJCeb27u45E1sIpInNGJzHzQfUSSfc3c3Jy0pKXdvl5ICzs7m9iJiV5TYZNGt8xFLlQIPD/N7pUrm8kmT4PBhW0cqUoi5uJiHUBMSwGS6fRuTyVzfpYtGa0TskBKbLNJ8RJECYsAA828dFy9mTm5MJnO5h4e5nYjYHSU2WaD5iCIFSGAgzJ4Nbm4QHW2e5R8XZ36PjjaXz56tSXEidkqTh7NA8xFFCpjgYPM+NRkz/ZOTwdXVXD5ggJIaETumxCYLMuYj3mskJiXF/L1T8xFF8oGAAPNrxgzzbxvu7vqNQ6QQ0K2oLNB8RJECzMXFPMtf/zFFCgUlNlmk+YgiIiL5nxKbLNJ8RBERkfxPc2zug+YjioiI5G8Gk+lOs0bsQ2xsLMWLF6dGjRq4u7uTlJREgwYNWLp0Ke4PMMs3KSn78xF37tzJuHHjMBqN2T6/iIiIPcv4+R0TE4Onp2eWj8v2ragzZ84wZcoU+vXrx6VLlwD45ptv+OWXX7LbZa5atmwZRqORX375hZiYGMLCwjK1SUtLy3J/mo8oIiKS/2Qrsfn++++pX78+e/fu5csvvyQ+Ph6Aw4cPM3Xq1BwNMKclJyeTkJBAyZIlCQsL49FHHyU4OJj69euzb98+2rRpw/r16y3te/XqZUmCPvzwQ+rWrUtgYKDl+tPT03nuueeoU6cOAQEBBAUFkZiYCMC3335LixYtCAoKonHjxuzYsSNTPH/99RcdOnSgfv36+Pv7M3To0Lz4MoiIiNilbM2xmTRpEjNmzOCFF17Aw8PDUt62bVsWLVqUY8HlpKFDh+Lu7k5kZCRBQUH06dOH8PBw9u7dS0REBL6+vvfs48UXX+TXX3+lQoUKpKSkkJSUxOHDh9m2bRu//PILRYoUISYmBmdnZ/73v/8RGhrKt99+i6enJ6dPn6Zly5ZERkZa9RkeHk7VqlX573//C8CVK1dy4/JFREQKhWyN2Bw9epQePXpkKi9btix///33AweVGzJuRf3999/4+PgwceJEAJo3b56lpAagXbt2DBw4kLfffpuzZ89SrFgxqlWrRmpqKsOGDWP58uWkpKRQpEgRtmzZwunTp2nVqhWBgYH06tWLIkWKcO7cOas+mzZtyjfffMOLL77IV1999UDzfkRERAq7bCU2JUqU4MKFC5nKIyIiqFSp0gMHlZscHR0JDg5my5YtABQrVixT/c1zbTJuKwGsXbuW2bNnk5KSQufOnfnss88oXrw4x44do3///vz666/4+/tz+vRpTCYTjz32GEaj0fKKjo6mZs2aVudr1qwZRqORJk2a8OWXX/Lwww/f11wfERER+Ue2Epsnn3ySiRMn8ueff2IwGEhPT+enn35iwoQJDBo0KKdjzHHbt2+/4yhNjRo12Lt3LwBnz55l165dAKSmpnLmzBkaNWrEhAkT6NWrF/v27eOvv/7i+vXrdOjQgddffx0fHx+OHz9Ox44d+e677zhy5Iil73379mU6X8bIT58+fVi4cCG//fabZc6SiIiI3J9szbF5/fXXefbZZ6lcuTJpaWnUrVuXtLQ0+vfvz5QpU3I6xhyRMccmNTUVb29vFi9ezLZt2zK1e/nll+nbty/169fHz8+PJk2aAOYVU8OGDePKlSs4Ojri5eXFsmXLiIqK4qmnniIlJYW0tDQeeeQRHn/8cZycnFi1ahWjRo0iISGB5ORkGjRowKpVq6zOt3PnTubPn4+DgwOpqanMmTOH4sWL58nXRERExN480D42586d49ixY8THx9OgQYNMt1nyg+yugxcRERHbye7P7wfaebhKlSpUqVLlQboQEZF84kE2HhXJL7Kc2LzwwgtZ7nT+/PnZCkZERPKe0fjPo2JSUsDJCbp0gYED9agYKXiynNhERERYfT506BCpqamWSbi//fYbDg4OBAUF5WyEIiKSa9asgcmTzQ/1LVrUnNQkJZkTna++Mj/cNzjY1lGKZF2WE5ubd82dP38+Hh4eLF++nJIlSwJw9epVhg4dSsuWLXM+ShERyXFGozmpuXEDKlUCg+GfupIl4eJFmDTJ/PBfjdxIQZGt5d7z5s1j1qxZlqQGoGTJksyYMYN58+ZluZ/Q0FAMBoPVq3bt2pb6xMREnn32WUqXLk2xYsUIDg7m4sWL2QlZRERuER5uHqkpV846qQHz53LlzPXh4baJTyQ7spXYxMbG8tdff2Uq/+uvv4iLi7uvvvz8/Lhw4YLllbFvDMD48ePZuHEjX3zxBd9//z3nz5+nZ8+e2QlZRERukpRknlNTtGjmpCaDwWCu37TJ3F6kIMjWqqgePXowdOhQ5s2bR+PGjQHYu3cvL7300n0nHo6OjpQvXz5TeUxMDB999BGrVq2ibdu2gPmxCHXq1GHPnj00bdo0O6GLiAjm1U8ZE4XvxskJkpPN7bVSSgqCbI3YLF68mMcff5z+/fvj7e2Nt7c3/fv3p1OnTrz33nv31depU6eoWLEi1apVIyQkxPIspYMHD5KSkkL79u0tbWvXrk2VKlXYvXt3dsIWEZH/5+5uTlpSUu7eLiUFnJ3N7UUKgmyN2BQtWpT33nuPOXPmcObMGQCqV69+3w9wbNKkCWFhYfj6+nLhwgWmTZtGy5YtOXbsGH/++SfOzs6UKFHC6phy5crx559/3rHPpKQkkm4aM42Njb2vmERECgMXF/OS7vBw80Th292OMpkgIcG8KkqjNVJQPNAGfe7u7vj7+2f7+Mcff9zyZ39/f5o0aYK3tzeff/45bm5u2epz1qxZTJs2LdsxiYgUFgMGmJd0X7yYeQKxyWQu9/AwtxMpKLKV2Dz66KMY7jTbDPNDJrOjRIkS1KpVi9OnT/PYY4+RnJzMtWvXrEZtLl68eNs5ORkmT55stZlgbGwslStXzlY8IiL2LDDQvE/NpEkQHf3PPjYpKeaRGg8Pc72WektBkq3EJjAw0OpzSkoKRqORY8eOMXjw4GwHEx8fz5kzZxg4cCBBQUE4OTmxbds2gv9/d6iTJ09y7tw5mjVrdsc+XFxccNGYqYhIlgQHm/epydh5ODkZXF3N5QMGKKmRgueBHoJ5q9DQUOLj45k7d26W2k+YMIGuXbvi7e3N+fPnmTp1KkajkePHj+Pl5cXo0aPZvHkzYWFheHp68vzzzwPw888/ZzkmPQRTRCRr9KwoyU9s8hDMWw0YMIDGjRtnObH5448/6NevH5cvX8bLy4sWLVqwZ88evLy8AFiwYAFFihQhODiYpKQkOnbseN+rrkRECgMfHx9cXFxwc3MjOTmZZ599lmefffa++nBxUUIjBV+OJja7d+/G1dU1y+0/++yzu9a7urry7rvv8u677z5oaHIf9FubSMG0evVqAgMD+f333/H396dly5aWBR7p6ekAFCmSrV0+RAqMbCU2t27CZzKZuHDhAgcOHODVV1/NkcAk7+kJvyL2wdvbG19fX/r374+vry/x8fFERUWxdetWFixYwPfff09KSgqenp4sXbrU8jDjr776ikmTJuHs7EynTp346KOPOHDgAD4+Pvj4+LB+/XrLHMtGjRoxd+5c2rRpw59//smYMWOIjIzkxo0bPPHEE8yYMcOGXwEpzLKVunt6elK8eHHLq1SpUrRp04bNmzczderUnI5R8sCaNdC7tzmxSUoCB4d/nvDbqxesXWvrCEUkq44ePcqvv/5KQEAAu3fv5pNPPuH48eNUqlSJiRMnsn//foxGI8888wxjx44F4NKlSwwbNox169Zx+PBhateuzeXLl7N0vsGDB/Pss8+yb98+IiIiOHDgAF988UVuXqLIHWVrxCYsLCyHwxBb0hN+RexD3759cXNzo2jRonz88cccO3YMNzc3ypUrZ2mzdetWFi5cSFxcHOnp6Vy5cgWAPXv24O/vb3kQ8eDBg3n66afvec7r16+zbds2qwcUx8fHc/LkyRy+OpGsyVZiU61aNfbv30/p0qWtyq9du0bDhg353//+lyPBSd7IeMLvrUkN/POE3+hoczslNiL5V8YcmwzHjh2jWLFils/nzp3jueeeY//+/VSvXp0jR47QqlWrLPXt6OhIWlqa5XNiYiJgnooA5sTofuZYiuSWbN2KioyMtPoHniEpKYno6OgHDkryjp7wK1J4xMTE4OTkRIUKFTCZTCxatMhS17RpU44cOWIZaQkPDyc5OdlSX6NGDfbu3QvAvn37LO2KFSvGo48+yuzZsy1tz58/zx9//JEXlySSyX2N2GzYsMHy52+//ZbixYtbPqelpbFt2zZ8fHxyLDjJfXrCr0jhUb9+fZ588kn8/PwoXbo03bt3t9SVLVuWDz/8kO7du+Pi4sJjjz1GsWLFLDu/z5gxg8GDB7NkyRKaNWuGn5+f5diVK1fywgsvUK9ePQwGA+7u7ixZsoSHHnooj69Q5D436MtYJmgwGLj1MCcnJ3x8fJg3bx7/+te/cjbKB6AN+u4uKcm8rXpSEpQqded2V66YdyONiFBiI2Kv4uLi8PDwAGD9+vVMnjyZEydO2DgqKazyZIO+jH0Qqlatyv79+ylTpsz9RSn5jp7wKyIZFi5cyOrVq0lLS8PT05OVK1faOiSR+5ajj1TIjzRic29Go3mp940bd37Cr5ubeUm4Jg+LiEheyPURm3feeYeRI0fi6urKO++8c9e2Y8aMyXIAYnt6wq+IiNiLLI/YVK1alQMHDlC6dGmqVq165w4Nhny13FsjNll3+LD1E36dnc23qfSEXxERyWvZ/fmtW1G57NZtyPODe8WU18+KSk1NZebMmXz66ac4Ojri6OhI48aNefPNNzl9+jRz5sxh9erVXLt2jcWLFzNp0qTcD0pERGwquz+/s7WPzfTp00lISMhUfuPGDaZPn56dLiUfcXExr5DKblKTmpp6X+2HDx/OgQMH2L17N8eOHSMiIoLHHnuMK1eu0KhRI1avXg2YN4C8ea8MERGRW2UrsZk2bRrx8fGZyhMSEpg2bdoDB1UYGAwGrl27ZvlcpkwZIiMj+euvv/Dx8WHPnj0ArFmzhoCAAG7cuEFcXBxPPfUUjRs3xt/fn5EjR1o20GrTpg0vvvgirVq1okqVKrz66qts3ryZFi1a4OPjw/z5863Ov3LlSoKCgqhRowZz5syxlB84cIDmzZvj7+9P48aN+emnnwDzpowZ+1mAect0w02zjA0GA1OnTuXhhx9m8uTJXLhwgQ4dOlC3bl06dOjAk08+SWhoaKavw+nTp/niiy9YtmwZJUuWtPTVu3dvqlWrxs6dOy0jS08//TRxcXEEBgbSqFEjDhw4QO3ata22HmjevDnffPPN/f+FiIiIXcjWIxVMJpPVD7UMhw8fptTdNkORe/Ly8mLFihWEhITw6aefMm7cOLZv346bmxsjR46kZcuWLF26FJPJxFNPPcXbb7/NSy+9BMDvv//Ojh07iI2NxcfHh6tXr/Ljjz9y/vx5fH19GTZsmCU5uXjxIgcOHODy5cs0bNiQRx55hEaNGtGzZ0+WLl1Kx44d2bVrF8HBwZw+fTpLsTs4OLB//34AevfuTbNmzZg2bRp//vkngYGBlmfQ3OzQoUPUrFkzS1sHLF68mMDAQIxGo6WsdOnSbN26lQ4dOhAREcFff/1Fp06dshSviIjYn/tKbEqWLInBYMBgMFCrVi2r5CYtLY34+PgsPTRN7q5ly5YMHz6c5s2b88knn1CrVi3AvGHW7t27LaMvN27cwMHBwXJcr169cHBwoGTJklSrVo1//etfGAwGKlWqhJeXF5GRkZbRj+HDh2MwGChTpgw9e/bku+++w8PDgyJFitCxY0cAWrRoQbly5TAajVnaQXTYsGGWP2/bto25c+cCUL58+VzbtHHs2LEsWrSIDh068O677/LMM8/cNukWEZHC4b4Sm7feeguTycSwYcOYNm2a1SMVnJ2d8fHxoVmzZjkepD1ycHC47QPlMkRERODl5UVUVJSlzGQysXbtWkuic6ubH0Dn4OCQ6fPd5r7cKRnIKL/TA/BudvPD9rLaf8OGDTl16hSXL1/O9FDVrOjZsycvv/wyERERbNiwwZJMiYhI4XRfc2wGDx7MkCFD2LFjB6NHj2bw4MGWV79+/ZTU3IebHyj35Zdfcv36dUvdokWLuHr1KocPH2bJkiWWeS7du3fnjTfesCQoV69ezfJtoluFhYUBcOXKFdatW0e7du3w9fUlPT2drVu3AvDzzz9bbiOVL18ek8nE8ePHAfjkk0/u2n/btm0t57h48SJff/31Hb8OwcHBDB8+3DLnKCOBu3XbAE9PT27cuGH1YD5HR0eefvppunXrRo8ePazmAYmISOGTrcnDrVu3xun/n5qYmJhIbGys1UusdezYkYceesjy+uOPP1iwYAFjx46lYcOGREREWEYrDh06xNy5c1m5ciVly5YlPDycgQMHcvnyZRYsWICbmxuBgYH4+/vTrl07IiMjsxWTl5cXQUFBNG7cmOeee47mzZvj7OzMl19+ydSpU/H392fcuHGsWbOGYsWK4ejoyMKFC/nXv/7Fww8/TEpKyl37f/vtt/nxxx+pW7cuISEhNGnS5I5Jx8cff0xAQABNmjTBz8+PunXr8t///jfTfK1SpUoxaNAg/P39adSokaV8+PDhREdH89xzz2XrayEiIvYjW/vYJCQk8PLLL/P5559z+fLlTPU337KwNVvvY1NY3bhxAycnJxwdHbl8+TJNmzYlPDycJk2a5Pi51qxZw/vvv8+2bdtyvG8REbGNPHkIZoaXXnqJHTt28P777zNw4EDeffddoqOjWbJkifYZEQBOnTrFoEGDMJlMJCcn88wzz+RKUtOpUyd+++031q1bl+N9i4hIwZOtEZsqVarwySef0KZNGzw9PTl06BA1atRgxYoVfPrpp2zevDk3Ys0WjdiIiIgUPHm68/CVK1eoVq0aYJ7QeeXKFcC8PPiHH37ITpciIiIiDyxbiU21atU4e/YsALVr1+bzzz8HYOPGjVZLwEVERETyUrYSm6FDh3L48GEAJk2axLvvvourqyvjx4/n5ZdfztEARURERLIqW5OHx48fb/lz+/bt+fXXXzl48CBlypQhPDw8x4ITERERuR/Zmjx8J4cPH6Zhw4Za7i0iIiIPJE8nD4uIiIjkR0psRERExG4osRERERG7cV+Th3v27HnX+oyHGIqIiIjYwn0lNvfao6Z48eIMGjTogQISERERya77SmyWLVuWW3GIiIiIPDDNsRERERG7ocRGRERE7IYSGxEREbEbSmxERETEbiixEREREbuhxEZERAqlpCS4csX8LvYjW0/3FhERKaiMRggPh02bICUFnJygSxcYOBACAmwdnTwojdiIiEihsWYN9O5tTmySksDBwfweHg69esHatQ9+Do0E2ZZGbEREpFAwGmHyZLhxAypVAoPhn7qSJeHiRZg0CWrUyN7IjUaC8geN2IiISKEQHg5xcVCunHVSA+bP5cqZ68PD77/vvBgJkqxRYiMiInYvKck8klK0aOakJoPBYK7ftOn+biPdOhJUqhR4eJjfK1Uyl0+aBIcP58ilyD0osREREbt3/fo/t4fuxskJkpPN7bMqN0eC5P4psREREbvn7m5OWlJS7t4uJQWcnc3tsyI3R4Ike/JNYjN79mwMBgPjxo2zlP35558MHDiQ8uXL4+7uTsOGDVmrG5UiInKfXFzME3kTEsBkun0bk8lc36WLuX1W5OZIkGRPvkhs9u/fz5IlS/D397cqHzRoECdPnmTDhg0cPXqUnj170qdPHyIiImwUqYiIFFQDBpjnvly8mDm5MZnM5R4e5nZZlVsjQZJ9Nk9s4uPjCQkJYenSpZQsWdKq7ueff+b555+ncePGVKtWjSlTplCiRAkOHjxoo2hFRKSgCgyE2bPBzQ2io817zcTFmd+jo83ls2ff39Ls3BoJkuyzeWLz7LPP0qVLF9q3b5+prnnz5qxevZorV66Qnp7OZ599RmJiIm3atLljf0lJScTGxlq9REREAIKDzUuzBw4EV1dISzO/DxxoLg8Ovv8+c2MkSLLPponNZ599xqFDh5g1a9Zt6z///HNSUlIoXbo0Li4ujBo1inXr1lGjRo079jlr1iyKFy9ueVWuXDm3whcRkTzWuXNnFi1alKk8ICAAf39/Vq5cec8+AgJgzhyIiIB9+8zvc+ZkfxO93BgJkuyzWWITFRXF2LFjWblyJa6urrdt8+qrr3Lt2jW+++47Dhw4wAsvvECfPn04evToHfudPHkyMTExlldUVFRuXYKIiOSx4cOHs2zZMquyAwcOcOHCBQ4ePEhISEiW+3JxMe81kxO3h3JjJEiyx2Ay3emuYO5av349PXr0wMHBwVKWlpaGwWCgSJEinDx5kho1anDs2DH8/Pwsbdq3b0+NGjVYvHhxls4TGxtL8eLFiYmJwdPTM8evQ0RE8k5KSgqVKlXiu+++syw4eeaZZyhatCh///03gYGBjBs3jpSUFF599VW2b99OcnIytWrVYs+ePbi5ufHXX39hMplwcXGhWLFi+Pv789lnn+Hs7ExoaCgnTpwgISGBM2fOUL58edasWUOpUqU4evQoo0ePJjIykqJFizJo0CCmTJnC4sWLiYuL46WXXgLMS7qvXzdPFM5ImoYMGWKJTbImuz+/bTZi065dO44ePYrRaLS8GjVqREhICEajkYSEBHOARaxDdHBwID093RYhi4iIjTk5OTFw4EA+/vhjABITE/n0008ZPny4Vbs5c+bg7u7Ovn37MBqN1K9fn2vXrrF69Wq6detGzZo1CQwMZOzYsVy8eNFqK5E9e/YQFhbG8ePHKVu2LEuWLAHAx8eHbdu24ejoyCeffMLatWvZs2cPTz/9tCWpgZwdCZL7Z7OHYHp4eFCvXj2rMnd3d0qXLk29evVISUmhRo0ajBo1irlz51K6dGnWr1/P1q1b+frrr20UtYiI2Nrw4cNp3bo1b775Jl9++SV16tShTp06Vm3Wr19PTEyMJWFJTk4mNTXVUt+1a1e2b99O6dKlcXd359///jdr1qxh165dNGvWjOXLl/Ppp59y/vx5tm3bRps2bahevTotWrTg3LlztG7dmrS0NDZu3MiWLVu4du0ab731Fnv27OHZZ58lLS2N1NRUnn32WUaPHm0V27Zt25gyZQqJiYkkJyfzwgsvMHz4cKKioggKCuKPP/7A2dkZMI/0NGjQgLFjxxISEsLJkydJTk6mcuXKfPTRR5QvXz6Xv9oFT759ureTkxObN29m0qRJdO3alfj4eGrUqMHy5cvp3LmzrcMTEREbqVu3LjVq1GDjxo18/PHHmUZrAEwmEwsXLqRDhw6WMh8fH/r27ctff/3FjRs3aN68OX369OGTTz4hKiqKb775hk8//ZRr164xcOBAXnjhBRYtWsQ333zD0KFDadGiBW3btiUmJob169czffp0vLy8uHbtmuUcs2bNYsKECfTr1w+Aq1evZoqtYcOG7Nq1CwcHB65cuUKDBg3o2LEjlStXJjAwkA0bNtCrVy/i4+PZsGED8+bNA+Ctt97Cy8sLMG9qGxoamuVpGYVJvkpsdu7cafW5Zs2a2mlYREQyGT58OK+//jqnTp1i/fr1meq7d+/OggULaNGiBUWLFiUhIYGUlBTWr1/PW2+9Rf369Tl+/DgTJ04EoHLlyvj6+lqOj4iIYObMmZw6dYrExESuXr1K7dq1LSNDkZGRbN26lVatWlmd99FHH+W1117j1KlTtG3blhYtWmSK7fLlywwfPpzffvsNR0dHLl++zLFjx3jooYcYOnQoy5Yto1evXnzxxRe0bduW0qVLA7Bq1SpWrFhBYmIiiYmJlClTJqe+nHbF5vvYiIiI3K++ffty8uRJevfuTbFixTLVT5w4kYcffpgmTZrg7+9P06ZNSU5OttQ7ODgQHBzMli1bACy3fsC8kKVnz57MnTuXV155hU6dOgHwwgsvsGzZMs6fP8/bb79N27ZtM5133LhxbNq0iQoVKvDKK6/wzDPPZGrz9NNP06JFC8s801q1apGYmAhAjx492LdvHxcuXCAsLIyhQ4cCsGvXLt555x02b97MsWPHmD9/vuUYsZavRmxERESywsPDg/j4eKuysLAwy58dHR2ZPn0606dPt5T5+PhYtZswYQK+vr5069bNMuoTGhpKbGwsS5YsoUqVKjz88MPExMQA4O/vz7Fjx/D39yc0NJTWrVtbjslw8uRJfH19eeqpp6hcuTKvvPJKptivXr2Kt7c3BoOBH374gcOHD1vqXF1d6d27N6GhoZw5c8aSVF29ehUPDw9Kly5NcnKyZUKzrd1uBZitKbEREZFCo2/fvri5uZGamoq3tzeLFy9m27ZtVm08PT2ZMWMGjRs3pkyZMjz55JNW9WPGjOGpp56iaNGiVskUwKJFi9i+fTvOzs44ODhY5sfcbPbs2TzzzDO89tprBAYG0qRJE6v6oUOH0rhxYyZOnGjZEqVTp06Eh4fj6+tL6dKlad++PdHR0TnwFckeoxHCw81PLM94CGiXLuZ9e2y9EaHN9rHJK9rHRkREJOesWQOTJ5t3Vy5a9J+HgCYkmB8dMXt2zmxImN2f3xqxERERkSwxGs1JzY0bUKkSGAz/1JUsaX4u1qRJUKOG7UZuNHlYREREsiQ83DxSU66cdVID5s/lypnrw8NtEx8osREREZEsSEoyz6kpWjRzUpPBYDDXb9pkbm8LSmxERETknq5f/2ei8N04OUFysrm9LSixERERkXtyd/9novDdpKSAs7O5vS0osREREZF7cnExL+lOSIA7rac2mcz1XbrYbl8bJTYiIiKSJQMGmJd0X7yYObkxmczlHh7mdraixEakgElKgitXbDcxT0QKr8BA8z41bm4QHW3+XhQXZ36PjjaXz55t2036tI+NSAGRn3f6FJHCIzjYvE9Nxvej5GRwdTWXDxhg++9H2nlYpADIq50+RcS++Pj44OLigpubG0lJSTRo0IClS5fifh8ze48dO8a//vUvIiMjM9Xl5rOiVq1aRUhIyH3//NatKJF87tadPkuVMiczpUqZP9+4Yd7p86bn6ImIWKxevRqj0cgvv/xCTExMpudbPQgXF/P3otyYKLxp06ZsHafERiSfKwg7fYpI/pecnExCQgIlS5YkLCyM7t27W+q+/vpr2rRpY/kcGhpKzZo1CQoK4rPPPrOUp6am0rFjRxo1aoSfnx/9+/fn+v9vWLNz507q1avHoEGDqFevHkFBQRiNRsuxK1asoEmTJjRs2JBWrVpZnmq+Z88egoKCCAwMpF69erz//vts3ryZzZs3A9CiRQs+/PDDLF+nEhspcArT5NmCstOniORfffv2JTAwkPLly1OkSBH69Olz1/abNm3iiy++4ODBgxw4cMDqFpSDgwOrVq3iwIEDHDt2jOLFi7Nw4UJL/S+//MLgwYM5duwYEydO5Mknn8RkMvHTTz/x6aef8sMPP3Do0CFmzpxJ//79AZg1axYTJkzAaDRy7NgxnnzySTp37kznzp0B2LVrFyNGjMjy9WrysBQYhXHybHZ2+rTV3hEikj+tXr2awMBAUlNTGTVqFBMnTqR+/fp3bL9t2zb69OljmdcyatQodu3aBYDJZGLBggVs2rSJ1NRUYmJiaN68ueVYHx8f2rVrB0CfPn0YOXIkUVFRfPXVVxw+fJgmTZpY2l65coUbN27w6KOP8tprr3Hq1Cnatm1LixYtHuh6NWIjBcKaNdC7tzmxSUoCBwfze3g49OoFa9faOsLcUVB2+hSR/M/R0ZHg4GC2bNmCo6MjaWlplrrExMQ7Hme4abh41apVbN++ne+//56jR48yYcKEex5rMBgwmUwMHjwYo9FoeV24cAE3NzfGjRvHpk2bqFChAq+88grPPPPMA12nEhvJ9wrz5NmCstOniBQM27dvx9fXlxo1anDkyBFu3LhBamoqq1atsrRp3749X3zxBXFxcZhMJj744ANL3dWrVylTpgyenp7ExcVlmogcGRnJjh07AFizZg3lypXjoYceolu3boSHh3Pu3DkA0tPTOXDgAAAnT56katWqPPXUU7zyyivs2bMHAA8Pj2xdoxIbyfcK++TZgrDTp4jkXxlzbOrVq8eJEyd4++23adq0KZ07d6ZevXq0adOGmjVrWtp37tyZXr160bBhQxo1akSVKlUsdYMGDSIhIQFfX18ef/xxWrZsaXUuPz8/wsLCqF+/PrNmzeLTTz/FYDDQsmVL3nzzTXr06EFAQAB+fn6WScmLFi3Cz8+PBg0aMGXKFObNm2eJG+5/8rD2sZF8LSnJvNNlUpJ5hOZOrlwxbxAVEWGfoxZr15pHpbSPjYjkVzt37mTcuHFWK6EeRHZ/fmvysORrmjxrlt93+hQRyS90K0ryNXucPNu5c2cWLVqUqTwgIIAvv/zyjscFBMCcOeZRqdde+5oyZdowZ46SGhHJH9q0aZNjozUPQomN5Gv2OHl2+PDhLFu2zKrswIEDXLhwga5du97zeBcX8+2nIvrfKyKSib41Sr5nb5Nnu3XrRlRUFEeOHLGUffzxx3Tr1o0OHToQFBSEn58fzz33HOnp6QCkpKTwzDPPULNmTRo3bmxZdQDm+9qBgYGWz8eOHcPHxweADz/8kMDAQMvLwcGB77//Pk+uU0TEFpTYSL4XGGieHOvmBtHR5onCcXHm9+hoc/ns2QXnloyTkxMDBw7k448/Bsz7R3z66ae8+OKLbNy4kYMHD3LkyBEiIyP5/PPPAfjggw84efIkv/zyC7t27eLQoUNZOteIESMse0b07t2b9u3b88gjj+TatYmI2JoSGykQgoPNm/QNHGieNJuWZn4fONBcXtBWBA0fPpyVK1eSnJzMl19+SZ06dfD29mbixIkEBATQoEEDDhw4YLlfvW3bNgYNGoSzszPOzs4MGzbsvs63YsUK1q5dy5o1a3B01JoBEbFf+g4nBUZAgPk1Y4Z59ZO7e8GYU3M7devWpUaNGmzcuJGPP/6Y4cOHM3/+fC5dusTevXtxdXXlhRdeuOOOnjfvBHqvHUS3b9/O9OnT+eGHH7K94ZWISEGhERvJN3x8fPD19SUwMJA6depYPTX2Zi4u5j1tCmpSk2H48OG8/vrr7Nu3j759+3L16lXKly+Pq6srf/75J1988YWlbfv27QkPDyclJYXk5GSrycfVqlXj999/56+//gLMozMZjh49yrBhw1i/fj0VKlTIu4sTEbERJTaSr6xevRqj0cgvv/xCTExMpu267yY9Pd0y2bYg6Nu3LydPnqR3794UK1aMsWPHsnfvXvz8/Bg4cCDt27e3tH3qqaeoWbMmdevWpUWLFlaThStWrMjLL79M48aNadq0KaVu2slw/vz5XL9+nZCQEMsE4oxtzEVE7JF2HpZ8w8fHh/Xr1xMYGEhiYiKPP/44Tz31FP3792fu3Ll8/vnnpKamUrZsWZYsWYK3tzehoaEcPXqU+Ph4oqKiWLp0KV26dGHs2LF8/fXXxMTE8M4779C5c2dbX56IiNyH7P781oiN5CsZzzQpX748RYoUoU+fPqxatYqTJ0+ye/duDh06REhIiNXTX3fv3s0nn3zC8ePHqVSpEjExMfj7+3Pw4EEWLVrE+PHjbXhFIiKSlzR5WPKV1atXExgYSGpqKqNGjWLixIlERUWxf/9+goKCAKwmyoJ5J99y5cpZPru6utKzZ08AmjVrxpkzZ/LuAkRExKY0YiP5kqOjI8HBwWzZsgWTycTkyZMt+7EcPXqUo0ePWtoWK1bM6lgXFxfLqiEHB4dMiZCIiNgvJTaSb23fvh1fX1+6d+/O4sWLuXLlCmDehTciIsLG0YmISH6kW1GSr/Tt2xc3NzdSU1Px9vZm8eLFVK5cmcuXL/Poo48CkJqayrBhw2jQoIGNoxURkfxGq6JEREQk39GqKBERESn0lNiIiIiI3VBiIyIiInZDiY2IiIjYDSU2IiJiJSkJrlwxv4sUNFruLSIiABiNEB4OmzZBSgo4OUGXLjBwIAQE2Do6kazRiI2IiLBmDfTubU5skpLAwcH8Hh4OvXrB2rW2jlAka/JNYjN79mwMBgPjxo2zKt+9ezdt27bF3d0dT09PWrVqxY0bN2wTpIiIHTIaYfJkuHEDKlWCUqXAw8P8XqmSuXzSJDh82NaRitxbvkhs9u/fz5IlS/D397cq3717N506daJDhw7s27eP/fv389xzz1GkSL4IW0TELoSHQ1wclCsH//+YNQuDwVweF2duJ5Lf2TxDiI+PJyQkhKVLl1KyZEmruvHjxzNmzBgmTZqEn58fvr6+9OnTBxcXFxtFKyJiX5KSzHNqihbNnNRkMBjM9Zs2aUKx5H82T2yeffZZunTpQvv27a3KL126xN69eylbtizNmzenXLlytG7dml27dt21v6SkJGJjY61eIiJye9ev/zNR+G6cnCA52dw+P9JKLslg08Tms88+49ChQ8yaNStT3f/+9z8AQkNDeeqpp9iyZQsNGzakXbt2nDp16o59zpo1i+LFi1telStXzrX4RUQKOnd3c9KSknL3dikp4Oxsbp+fGI0wYQIEBkLjxub3CRM0H6gws1liExUVxdixY1m5ciWurq6Z6tPT0wEYNWoUQ4cOpUGDBixYsABfX18+/vjjO/Y7efJkYmJiLK+oqKhcuwYRkYLOxcW8pDshAe70SGSTyVzfpYu5fX6hlVxyOzZLbA4ePMilS5do2LAhjo6OODo68v333/POO+/g6OhIuXLlAKhbt67VcXXq1OHcuXN37NfFxQVPT0+rl4iI3NmAAeZVUBcvZk5uTCZzuYeHuV1+oZVccic2S2zatWvH0aNHMRqNllejRo0ICQnBaDRSrVo1KlasyMmTJ62O++233/D29rZR1CIi9icwEGbPBjc3iI42z1WJizO/R0eby2fPzl+b9Gkll9yJzXYe9vDwoF69elZl7u7ulC5d2lL+0ksvMXXqVAICAggMDGT58uX8+uuvrFmzxhYhi4jYreBgqFHjn52Hk5PB1dVcPmBA/kpq7ncl14wZ+esWmuQum6+Kuptx48YxefJkxo8fT0BAANu2bWPr1q1Ur17d1qGJiNiUj48PRqPRqmzEiBHs2LHjnseGhoZm2gwVzMmLu3so7dqNY98+eO21DaSmjs9XSQ3Yz0ouyR356llRO3fuzFQ2adIkJk2alPfBiIgUMB9++GGO9OPgYJ6rEhzcjeDgbjnSZ07KWMl1r6XdKSnmUaf8tpJLcle+HrEREZGsa9OmDevXrwdgyJAhvPXWW5a6CRMmEBoaavkcFRVF27ZtqV27Nl27duXy5cuZ+gsLC6N79+4A/Pnnnzz66KMEBQXh5+fHc889Z1m9GhYWRvv27enXrx/169enUaNGli077nZcdhXklVyS+5TYiIgUQj/++COrVq3i119/pXLlykyePPmu7UuUKMHGjRs5ePAgR44cITIyks8//9xSv3//fl5//XWOHj1K+/bteeONN7J0XHYVxJVckjeU2IiIFEJdunShfPnyAIwcOZLvvvvuru3T09OZOHEiAQEBNGjQgAMHDljN8WnWrBlVq1a1/PnMmTNZOi67CuJKLskb+WqOjYiI5AxHR0fS0tIsnxMTEylWrNgd2xvutLzo/82fP9/yqBtXV1deeOEFEhMTLfU3b7Tq4OBAampqlo57EAVpJZfkHY3YiIjYoRo1arBv3z4ALl++zObNm63qN2/ezMWLFwHzpONbn9d3q6tXr1K+fHlcXV35888/+eKLL7IUx72O27BhA4GBgVavSpUq3XZH+tsJCIA5cyAiAvbtM7/PmZP1pCYyMpISJUpkrbEUCBqxEREpoDp27IjTTWueXV1dLSMvI0eOpFevXtSpU4dq1arRtGlTq2NbtmxJ//79iY6OpmbNmoSFhd32HBn9jR07ll69euHn50fFihXvmQhluNdx3bp1o1u3f1ZeXbt2jYcffpjp06dnqf8MLi53niSclpaGg4PDffUnBZfBZLrTnHL7EBsbS/HixYmJidHjFUTErvn5+fHBBx/wyCOP5Eh/b775JqdOnWLp0qU50t+9pKen07VrVypXrszixYs5evQoo0ePJiEhgcTERPr378+UKVMAiIuLY8SIERw+fBgvLy/q1q1LUlISYWFhhIWFsXz5ckqVKsVvv/3GBx98gKOjIxMnTiQ2Npa0tDReeeUVevfuTWRkJIGBgYwYMYL//ve/pKWl8fbbb2c5cZPck92f3xqxERGxA76+vtSpUyfTyEx2/fvf/2bdunWsXLkyR/rLiqlTp3LlyhXWrVsHmDch3LZtGy4uLty4cYPmzZvTvn17mjZtyvTp03Fzc+PEiRPEx8fTvHlzgoKCLH3t3buXiIgIfH19uXbtGo8++iibN2+mQoUK/P333zRs2JDmzZsDEBMTQ506dZg7dy579uyhW7dunDlzBg8Pjzy7dsk5SmxEROzArc/Ve1AzZ85k5syZOdrn3Xz11Vd89NFHHDhwAGdnZwBu3LjBM888g9FopEiRIkRFRWE0GmnatCnbtm1jwYIFGAwGPDw86Nu3L6dPn7b017x5c3x9fQH4+eef+d///sfjjz9udc6TJ09SrVo1HB0dGTJkCABNmzalYsWKRERE0KpVq7y5eMlRSmxERMSmTp48yfDhw1m/fj0VK1a0lL/yyiuUKVOGiIgIHB0d6dmz5x1XVN26quvmFWAmkwk/Pz9+/vnnTMdFRkZmqT8pOLQqSkREbCYuLo4ePXowbdo0WrRoYVV39epVHnroIRwdHTl58iRbt2611LVt25bly5djMpmIj4+/66Z/zZs35+zZs1Z79RiNRpKTkwFITU1lxYoVAOzbt4/z588TGBiYg1cpeUkjNiIiYjPvvvsuJ0+eZOnSpZkmKb/77ruMGjWK5cuXU716ddq2bWup+89//sPw4cOpU6cOZcqUISAg4I7LtkuWLMmmTZuYMGECL774IikpKVSpUsXy+InixYtz7NgxAgICSE1NZdWqVZpfU4BpVZSIiBQ4KSkppKWl4erqyvXr1+nYsSPPP/88ffv2tXVokkO0KkpERAqNq1ev8vjjj5OWlkZiYiJPPPEEffr0sXVYkg8osRERkQKnbNmyHDx40NZhSD6kycMiIiJiN5TYiIiIiN1QYiMiIiJ2Q4mNiIiI2A0lNiIiImI3lNiIiIiI3VBiIyIiInZDiY2IiIjYDSU2IiIiYjeU2IiIiIjdUGIjIiIidkOJjYiIiNgNJTYiIiJiN5TYiIiIiN1QYiMiIiJ2Q4mNiIiI2A0lNiIiImI3lNiIiIiI3VBiIyIiInZDiY2IiIjYDSU2IiIiYjeU2IiIiIjdUGIjIiIidkOJjYiIiNgNJTYiIiJiN5TYiIiIiN1QYiMiIiJ2Q4mNiIiI2A0lNiIiImI3lNiIiIiI3cg3ic3s2bMxGAyMGzcuU53JZOLxxx/HYDCwfv36PI9NRAqfpCS4csX8LiIFh6OtAwDYv38/S5Yswd/f/7b1b731FgaDIY+jEpHCyGiE8HDYtAlSUsDJCbp0gYEDISDA1tGJyL3YfMQmPj6ekJAQli5dSsmSJTPVG41G5s2bx8cff2yD6ESkMFmzBnr3Nic2SUng4GB+Dw+HXr1g7VpbRygi92LzxObZZ5+lS5cutG/fPlNdQkIC/fv3591336V8+fJZ6i8pKYnY2Firl4jIvRiNMHky3LgBlSpBqVLg4WF+r1TJXD5pEhw+bOtIReRubJrYfPbZZxw6dIhZs2bdtn78+PE0b96cJ554Ist9zpo1i+LFi1telStXzqlwRcSOhYdDXByUKwe33vk2GMzlcXHmdiKSf9kssYmKimLs2LGsXLkSV1fXTPUbNmxg+/btvPXWW/fV7+TJk4mJibG8oqKicihiEbFXSUnmOTVFi2ZOajIYDOb6TZs0oVgkP7NZYnPw4EEuXbpEw4YNcXR0xNHRke+//5533nkHR0dHtm7dypkzZyhRooSlHiA4OJg2bdrcsV8XFxc8PT2tXmJftFpFctr16/9MFL4bJydITja3F5H8yWarotq1a8fRo0etyoYOHUrt2rWZOHEiZcqUYdSoUVb19evXZ8GCBXTt2jUvQ5V8QqtVJLe4u5v/Pd0rWU5JAVdXc3sRyZ9slth4eHhQr149qzJ3d3dKly5tKb/dhOEqVapQtWrVPIlR8o81a8wTO+PizLcDMn4IhYfDV1/B7NkQHGzrKKWgcnExJ8nh4VCy5O1vR5lMkJBg/nfm4pL3MYpI1th8VZTIvWi1iuSFAQPM/64uXjQnMTczmczlHh7mdiKSf+WLDfoy7Ny58671plu/20ihkLFapVKlO69WiY42t9MtKcmuwEDzyN+kSeZ/Txkjgykp5pEaDw9zvf6NieRvGrGRfE2rVSQvBQebb3sOHGieS5OWZn4fONBcrtudIvmfEhvJ17RaRQC+/PJLgoKCCAwMpHbt2rRt25b09PQc63/nzp0YDAbGjh1LQADMmQMREdCo0WBOnDAQEmK875GaAwcO0Ldv3xyLUUSyJl/dihK5lVaryIULFxg5ciQHDx7E29sbgEOHDuX48+Nq1qzJxo0bmTNnDs7OziQlxXLgwE9UqlTpvvtKTU2lUaNGrF69OkdjFJF704iN5GsZq1USEjJP6MyQsVqlSxetVrFHFy9exMHBgVKlSlnKGjZsaElsJkyYwMMPP0xgYCCtWrXi5MmTlnYGg4HXX3+dxo0bU7VqVZYtW3bH8xQtWpR27drx1VdfAead0YODgy17aAHMnz/fcq6HH36Y3bt3W+p8fHyYOHEijRs3ZvDgwezcuZPAwEAAIiMjKVGiBFOnTiUoKIgaNWqwefNmy7HffvstDRs2xN/fn9atW3P8+PEH+6KJFGJKbCTf02qVws3f358WLVrg7e1Njx49mDNnDtHR0Zb6iRMnsn//foxGI8888wxjx461Ot7FxYV9+/bxzTffMGbMGFJTU+94rqFDh1oeuLts2TKGDRtmVT9w4EDLuRYuXMjQoUOt6i9fvszevXtZuXJlpr5jYmLw9/fn4MGDLFq0iPHjxwNw6dIl+vfvz/Llyzly5AgjR46kV69eWiwhkk1KbCTfy1it4uZmXq1y5Yp5ldSVK+bPbm5arWLPihQpwtq1a/n555/p1KkTP/30E35+fpw+fRqArVu30qxZM+rVq8f06dMxGo1Wx4eEhABQu3ZtHB0d+fPPP+94rubNm3Pu3Dm+/fZbHBwc8PX1taqPiIigdevW1KtXj6effpqTJ09y48YNS/2QIUPueIvM1dWVnj17AtCsWTPOnDkDwN69e6lfvz7169e3xHv+/Hmr5E1Esk5zbKRACA6GGjX+2Xk4Odk8pyY42DxSo6TG/tWuXZvatWszatQoOnXqxIYNG+jVqxfPPfcc+/fvp3r16hw5coRWrVpZHXfzs+gcHBzuOmIDMGjQIAYMGMDs2bOtypOTk+nZsyc7duzg4YcfJjY2luLFi5OUlISbmxsAxYoVu2O/Li4ulqTHwcGBtLS0+7p+EckaJTaS63x8fEhISCA6Ohqn/1/etGPHDtq2bcvYsWOz/KDTgADza8YM8+ond3fNqSkMoqOjiYyM5JFHHgHg6tWrnD17lurVqxMTE4OTkxMVKlTAZDKxaNGiBz7f0KFDMZlMmVY0JSYmkpycTJUqVQBYuHDhA58LoGnTphw9epRjx45Rr149PvvsMypVqpStScsiosRG8kiVKlXYsGEDwf+/EchHH31Eo0aNstWXi4sSmsIkNTWV6dOnc/bsWYoWLUpqaiqDBw/miSeeAODJJ5/Ez8+P0qVL07179wc+X9myZZk0aVKmck9PT2bMmEHjxo0pU6YMTz755AOfC8DLy4uVK1cyaNAgUlNTKVmyJF988UWOr/oSKSwMJjufoZYxXBwTE6MnfduIj48PL730Eps3b2bTpk3ExMQQFBREv379iIuLY968eUyaNIlvvvkGgEcffZR58+bh7OzMkCFDcHFx4fTp00RFRVl+o3V2dmbbtm1MmTLF8pv0Cy+8wPDhwwHzRM0XX3yRPXv24ODgQFBQEB9//DHx8fGMGTOGffv2AdC7d2+mTp1qs6+NiIjcXnZ/fmvysOSJRx55hMjISM6fP8+nn35K7969cXBwAOCDDz5g//79HDx4EKPRyJkzZ1iwYIHlWKPRyMaNGzlx4gQXL15k7dq1gHnJ765du4iIiODHH39k+vTp/PHHHwCMGzcOZ2dnjhw5wuHDh3njjTcAeO2110hKSuLIkSPs3buX9evXa68RERE7osRG8szAgQMJCwvj448/tlpG+91331lGZhwdHXnqqafYunWrpb5Hjx4ULVoUBwcHGjdubFlNcvnyZXr37k29evVo27Ytly9f5tixYwB8/fXXTJgwgSJFzP/Evby8LOd66qmnKFKkCO7u7gwaNMjqXCIiUrBpjo3kmUGDBtGwYUNq1apFzZo179ju1rkFd1rV8vTTT9O5c2fWrl2LwWCgYcOGJCYm3ldMmscgImJfNGIjeaZixYrMmjXLclsoQ/v27fnkk09ITk4mNTWVDz/8kA4dOtyzv6tXr+Lt7Y3BYOCHH37g8OHDlrpu3boxd+5cy/OE/vrrL8u5PvroI0wmE9evX2fFihVZOpeIiBQMSmwkTw0dOpRmzZpZlY0cOZKGDRvSsGFDAgMD8fHxYdy4cffsa/bs2UyaNInAwEA+/vhjmjRpYqlbsGABSUlJ1K9fn8DAQF555RUAXn31VZycnKhfvz5NmjShW7du9OnTJ0evUUREbEerokRERCTf0aooERERKfTsfvJwxoBUbGysjSMRERGRrMr4uX2/N5bsPrGJi4sDoHLlyjaORERERO5XXFwcxYsXz3J7u59jk56ezvnz5/Hw8MjS0t7Y2FgqV65MVFRUoZmTo2suHNcMhfO6C+M1Q+G8bl2zfV2zyWQiLi6OihUrWvYkywq7H7EpUqQIDz300H0f5+npaXf/SO5F11x4FMbrLozXDIXzunXN9uN+RmoyaPKwiIiI2A0lNiIiImI3lNjcwsXFhalTp+Li4mLrUPKMrrnwKIzXXRivGQrndeuaBQrB5GEREREpPDRiIyIiInZDiY2IiIjYDSU2IiIiYjeU2IiIiIjdUGJzG7Nnz8ZgMDBu3Dhbh5LroqOjGTBgAKVLl8bNzY369etz4MABW4eVa9LS0nj11VepWrUqbm5uVK9enddee+2+n0WS3/3www907dqVihUrYjAYWL9+vVW9yWTiP//5DxUqVMDNzY327dtz6tQp2wSbQ+52zSkpKUycOJH69evj7u5OxYoVGTRoEOfPn7ddwDngXn/PN3v66acxGAy89dZbeRZfbsnKdZ84cYJu3bpRvHhx3N3defjhhzl37lzeB5tD7nXN8fHxPPfcczz00EO4ublRt25dFi9ebJtgbUyJzS3279/PkiVL8Pf3t3Uoue7q1as88sgjODk58c0333D8+HHmzZtHyZIlbR1arnnjjTd4//33WbRoESdOnOCNN97gzTffZOHChbYOLUddv36dgIAA3n333dvWv/nmm7zzzjssXryYvXv34u7uTseOHUlMTMzjSHPO3a45ISGBQ4cO8eqrr3Lo0CG+/PJLTp48Sbdu3WwQac65199zhnXr1rFnzx4qVqyYR5Hlrntd95kzZ2jRogW1a9dm586dHDlyhFdffRVXV9c8jjTn3OuaX3jhBbZs2UJ4eDgnTpxg3LhxPPfcc2zYsCGPI80HTGIRFxdnqlmzpmnr1q2m1q1bm8aOHWvrkHLVxIkTTS1atLB1GHmqS5cupmHDhlmV9ezZ0xQSEmKjiHIfYFq3bp3lc3p6uql8+fKmOXPmWMquXbtmcnFxMX366ac2iDDn3XrNt7Nv3z4TYPr999/zJqhcdqdr/uOPP0yVKlUyHTt2zOTt7W1asGBBnseWm2533X379jUNGDDANgHlgdtds5+fn2n69OlWZQ0bNjT9+9//zsPI8geN2Nzk2WefpUuXLrRv397WoeSJDRs20KhRI3r37k3ZsmVp0KABS5cutXVYuap58+Zs27aN3377DYDDhw+za9cuHn/8cRtHlnfOnj3Ln3/+afXvvHjx4jRp0oTdu3fbMLK8FRMTg8FgoESJErYOJdekp6czcOBAXnrpJfz8/GwdTp5IT09n06ZN1KpVi44dO1K2bFmaNGly19t09qB58+Zs2LCB6OhoTCYTO3bs4LfffqNDhw62Di3PKbH5f5999hmHDh1i1qxZtg4lz/zvf//j/fffp2bNmnz77beMHj2aMWPGsHz5cluHlmsmTZrEk08+Se3atXFycqJBgwaMGzeOkJAQW4eWZ/78808AypUrZ1Verlw5S529S0xMZOLEifTr188uHxyY4Y033sDR0ZExY8bYOpQ8c+nSJeLj45k9ezadOnXiv//9Lz169KBnz558//33tg4v1yxcuJC6devy0EMP4ezsTKdOnXj33Xdp1aqVrUPLc3b/dO+siIqKYuzYsWzdurVA34O9X+np6TRq1IjXX38dgAYNGnDs2DEWL17M4MGDbRxd7vj8889ZuXIlq1atws/PD6PRyLhx46hYsaLdXrNYS0lJoU+fPphMJt5//31bh5NrDh48yNtvv82hQ4cwGAy2DifPpKenA/DEE08wfvx4AAIDA/n5559ZvHgxrVu3tmV4uWbhwoXs2bOHDRs24O3tzQ8//MCzzz5LxYoVC81diAwascH8DeDSpUs0bNgQR0dHHB0d+f7773nnnXdwdHQkLS3N1iHmigoVKlC3bl2rsjp16hTolQP38tJLL1lGberXr8/AgQMZP358oRqpK1++PAAXL160Kr948aKlzl5lJDW///47W7dutevRmh9//JFLly5RpUoVy/e133//nRdffBEfHx9bh5drypQpg6OjY6H63nbjxg1eeeUV5s+fT9euXfH39+e5556jb9++zJ0719bh5TmN2ADt2rXj6NGjVmVDhw6ldu3aTJw4EQcHBxtFlrseeeQRTp48aVX222+/4e3tbaOIcl9CQgJFiljn8w4ODpbf8gqDqlWrUr58ebZt20ZgYCAAsbGx7N27l9GjR9s2uFyUkdScOnWKHTt2ULp0aVuHlKsGDhyY6Tf1jh07MnDgQIYOHWqjqHKfs7MzDz/8cKH63paSkkJKSkqh/96WQYkN4OHhQb169azK3N3dKV26dKZyezJ+/HiaN2/O66+/Tp8+fdi3bx8ffPABH3zwga1DyzVdu3Zl5syZVKlSBT8/PyIiIpg/fz7Dhg2zdWg5Kj4+ntOnT1s+nz17FqPRSKlSpahSpQrjxo1jxowZ1KxZk6pVq/Lqq69SsWJFunfvbrugH9DdrrlChQr06tWLQ4cO8fXXX5OWlmaZT1SqVCmcnZ1tFfYDudff863Jm5OTE+XLl8fX1zevQ81R97rul156ib59+9KqVSseffRRtmzZwsaNG9m5c6ftgn5A97rm1q1b89JLL+Hm5oa3tzfff/89n3zyCfPnz7dh1DZi62VZ+VVhWO5tMplMGzduNNWrV8/k4uJiql27tumDDz6wdUi5KjY21jR27FhTlSpVTK6urqZq1aqZ/v3vf5uSkpJsHVqO2rFjhwnI9Bo8eLDJZDIv+X711VdN5cqVM7m4uJjatWtnOnnypG2DfkB3u+azZ8/etg4w7dixw9ahZ9u9/p5vZS/LvbNy3R999JGpRo0aJldXV1NAQIBp/fr1tgs4B9zrmi9cuGAaMmSIqWLFiiZXV1eTr6+vad68eab09HTbBm4DBpPJzrZcFRERkUJLk4dFRETEbiixEREREbuhxEZERETshhIbERERsRtKbERERMRuKLERERERu6HERkREROyGEhsRKbAiIyMxGAwYjcZc6d9gMLB+/fpc6VtEcocSGxHJtiFDhtj0MQyVK1fmwoULlkef7Ny5E4PBwLVr12wWk4jYlp4VJSIFloODg90/kVxE7o9GbEQkV3z//fc0btwYFxcXKlSowKRJk0hNTbXUt2nThjFjxvDyyy9TqlQpypcvT2hoqFUfv/76Ky1atMDV1ZW6devy3XffWd0euvlWVGRkJI8++igAJUuWxGAwMGTIEAB8fHx46623rPoODAy0Ot+pU6do1aqV5Vxbt27NdE1RUVH06dOHEiVKUKpUKZ544gkiIyMf9EslIjlIiY2I5Ljo6Gg6d+7Mww8/zOHDh3n//ff56KOPmDFjhlW75cuX4+7uzt69e3nzzTeZPn26JaFIS0uje/fuFC1alL179/LBBx/w73//+47nrFy5MmvXrgXg5MmTXLhwgbfffjtL8aanp9OzZ0+cnZ3Zu3cvixcvZuLEiVZtUlJS6NixIx4eHvz444/89NNPFCtWjE6dOpGcnHw/Xx4RyUW6FSUiOe69996jcuXKLFq0CIPBQO3atTl//jwTJ07kP//5D0WKmH+n8vf3Z+rUqQDUrFmTRYsWsW3bNh577DG2bt3KmTNn2Llzp+V208yZM3nsscdue04HBwdKlSoFQNmyZSlRokSW4/3uu+/49ddf+fbbb6lYsSIAr7/+Oo8//rilzerVq0lPT+fDDz/EYDAAsGzZMkqUKMHOnTvp0KHD/X2RRCRXKLERkRx34sQJmjVrZkkAAB555BHi4+P5448/qFKlCmBObG5WoUIFLl26BJhHXSpXrmw1h6Zx48a5Fm/lypUtSQ1As2bNrNocPnyY06dP4+HhYVWemJjImTNnciUuEbl/SmxExGacnJysPhsMBtLT03P8PEWKFMFkMlmVpaSk3Fcf8fHxBAUFsXLlykx1Xl5eDxSfiOQcJTYikuPq1KnD2rVrMZlMllGbn376CQ8PDx566KEs9eHr60tUVBQXL16kXLlyAOzfv/+uxzg7OwPm+Tk38/Ly4sKFC5bPsbGxnD171ireqKgoLly4QIUKFQDYs2ePVR8NGzZk9erVlC1bFk9Pzyxdg4jkPU0eFpEHEhMTg9FotHqNHDmSqKgonn/+eX799Ve++uorpk6dygsvvGCZX3Mvjz32GNWrV2fw4MEcOXKEn376iSlTpgBY3eK6mbe3NwaDga+//pq//vqL+Ph4ANq2bcuKFSv48ccfOXr0KIMHD8bBwcFyXPv27alVqxaDBw/m8OHD/Pjjj5kmKoeEhFCmTBmeeOIJfvzxR86ePcvOnTsZM2YMf/zxR3a+dCKSC5TYiMgD2blzJw0aNLB6vfbaa2zevJl9+/YREBDA008/zfDhwy2JSVY4ODiwfv164uPjefjhhxkxYoQl2XB1db3tMZUqVWLatGlMmjSJcuXK8dxzzwEwefJkWrduzb/+9S+6dOlC9+7dqV69uuW4IkWKsG7dOm7cuEHjxo0ZMWIEM2fOtOq7aNGi/PDDD1SpUoWePXtSp04dhg8fTmJiokZwRPIRg+nWG88iIvnUTz/9RIsWLTh9+rRVYiIikkGJjYjkW+vWraNYsWLUrFmT06dPM3bsWEqWLMmuXbtsHZqI5FOaPCwi+VZcXBwTJ07k3LlzlClThvbt2zNv3jxbhyUi+ZhGbERERMRuaPKwiIiI2A0lNiIiImI3lNiIiIiI3VBiIyIiInZDiY2IiIjYDSU2IiIiYjeU2IiIiIjdUGIjIiIidkOJjYiIiNiN/wOGp5DFD+tM8QAAAABJRU5ErkJggg==\n" + }, + "metadata": {} + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Closing remarks\n", + "\n", + "Typically, you can leave the Store open while your application runs and never close it. However, for this Jupyter Notebook, we also include a close so you can start from scratch with opening the store again. This avoids opening the same store multiple times." + ], + "metadata": { + "id": "C-jTEeR19Vx0" + } + }, + { + "cell_type": "code", + "source": [ + "# Uncomment this to close the store, if you want to open it again:\n", + "# store.close()" + ], + "metadata": { + "id": "DRfBiI60-OvC" + }, + "execution_count": 10, + "outputs": [] + } + ] +} diff --git a/example/vectorsearch-cities/cities.csv b/example/vectorsearch-cities/cities.csv new file mode 100644 index 0000000..d0a609d --- /dev/null +++ b/example/vectorsearch-cities/cities.csv @@ -0,0 +1,213 @@ +Abuja, 9.0765, 7.3986 +Accra, 5.6037, -0.1870 +Addis Ababa, 9.0084, 38.7813 +Algiers, 36.7529, 3.0420 +Amman, 31.9632, 35.9306 +Amsterdam, 52.3667, 4.8945 +Ankara, 39.9334, 32.8597 +Antananarivo, -18.8792, 47.5079 +Apia, -13.8330, -171.7667 +Ashgabat, 37.9601, 58.3261 +Asmara, 15.3229, 38.9251 +Astana, 51.1796, 71.4475 +Asunción, -25.2637, -57.5759 +Athens, 37.9795, 23.7162 +Avarua, -21.2079, -159.7750 +Baghdad, 33.3152, 44.3661 +Baku, 40.4093, 49.8671 +Bamako, 12.6530, -7.9864 +Bandar Seri Begawan, 4.9031, 114.9398 +Bangkok, 13.7563, 100.5018 +Bangui, 4.3947, 18.5582 +Banjul, 13.4549, -16.5790 +Basseterre, 17.3026, -62.7177 +Beijing, 39.9042, 116.4074 +Beirut, 33.8889, 35.4944 +Belgrade, 44.7866, 20.4489 +Belmopan, 17.2510, -88.7590 +Berlin, 52.5200, 13.4050 +Bern, 46.9480, 7.4474 +Bishkek, 42.8746, 74.5698 +Bissau, 11.8636, -15.5842 +Bogotá, 4.7109, -74.0721 +Brasília, -15.8267, -47.9218 +Bratislava, 48.1486, 17.1077 +Brazzaville, -4.2634, 15.2429 +Bridgetown, 13.1132, -59.5988 +Brussels, 50.8503, 4.3517 +Bucharest, 44.4268, 26.1025 +Budapest, 47.4979, 19.0402 +Buenos Aires, -34.6037, -58.3816 +Bujumbura, -3.3818, 29.3622 +Cairo, 30.0444, 31.2357 +Canberra, -35.2809, 149.1300 +Caracas, 10.4806, -66.9036 +Castries, 14.0101, -60.9874 +Chisinau, 47.0105, 28.8638 +Colombo, 6.9271, 79.8612 +Conakry, 9.6412, -13.5784 +Copenhagen, 55.6761, 12.5683 +Dakar, 14.7167, -17.4677 +Damascus, 33.5131, 36.2919 +Dhaka, 23.8103, 90.4125 +Dili, -8.5569, 125.5603 +Djibouti, 11.5890, 43.1456 +Dodoma, -6.1748, 35.7469 +Doha, 25.2854, 51.5310 +Dublin, 53.3498, -6.2603 +Dushanbe, 38.5868, 68.7841 +Freetown, 8.4840, -13.2299 +Funafuti, -8.5210, 179.1962 +Gaborone, -24.6282, 25.9231 +Georgetown, 6.8013, -58.1550 +Gibraltar, 36.1408, -5.3536 +Guatemala City, 14.6349, -90.5069 +Hanoi, 21.0278, 105.8342 +Harare, -17.8252, 31.0335 +Havana, 23.1136, -82.3666 +Helsinki, 60.1699, 24.9384 +Honiara, -9.4376, 159.9720 +Islamabad, 33.6844, 73.0479 +Jakarta, -6.2088, 106.8456 +Juba, 4.8594, 31.5713 +Kabul, 34.5553, 69.2075 +Kampala, 0.3476, 32.5825 +Kathmandu, 27.7172, 85.3240 +Khartoum, 15.5007, 32.5599 +Kiev, 50.4501, 30.5234 +Kigali, -1.9441, 30.0619 +Kingston, 17.9710, -76.7924 +Kingstown, 13.1467, -61.2121 +Kinshasa, -4.4419, 15.2663 +Kuala Lumpur, 3.1390, 101.6869 +Kuwait City, 29.3759, 47.9774 +La Paz, -16.4897, -68.1193 +Libreville, 0.4162, 9.4673 +Lilongwe, -13.9626, 33.7741 +Lima, -12.0464, -77.0428 +Lisbon, 38.7223, -9.1393 +Ljubljana, 46.0569, 14.5058 +Lomé, 6.1319, 1.2228 +London, 51.5072, -0.1276 +Luanda, -8.8399, 13.2894 +Lusaka, -15.3875, 28.3228 +Luxembourg City, 49.6116, 6.1319 +Madrid, 40.4168, -3.7038 +Majuro, 7.1164, 171.1859 +Malabo, 3.7508, 8.7839 +Male, 4.1755, 73.5093 +Mamoudzou, -12.7871, 45.2750 +Managua, 12.1364, -86.2514 +Manama, 26.2285, 50.5860 +Manila, 14.5995, 120.9842 +Maputo, -25.8918, 32.6051 +Maseru, -29.2976, 27.4854 +Mbabane, -26.3054, 31.1367 +Melekeok, 7.4874, 134.6265 +Mexico City, 19.4326, -99.1332 +Minsk, 53.9045, 27.5615 +Mogadishu, 2.0469, 45.3182 +Monaco, 43.7325, 7.4189 +Monrovia, 6.3005, -10.7974 +Montevideo, -34.9011, -56.1645 +Moroni, -11.7022, 43.2551 +Moscow, 55.7558, 37.6173 +Muscat, 23.5859, 58.4059 +Nairobi, -1.2921, 36.8219 +Nassau, 25.0478, -77.3554 +Naypyidaw, 19.7633, 96.0785 +New Delhi, 28.6139, 77.2090 +Ngerulmud, 7.5004, 134.6249 +Niamey, 13.5122, 2.1254 +Nicosia, 35.1725, 33.365 +Nicosia Northern Cyprus, 35.19, 33.363611 +Nouakchott, 18.0735, -15.9582 +Nuku'alofa, -21.1393, -175.2049 +Nuuk, 64.1836, -51.7214 +Oranjestad, 12.5092, -70.0086 +Oslo, 59.9139, 10.7522 +Ottawa, 45.4215, -75.6972 +Ouagadougou, 12.3714, -1.5197 +Pago Pago, -14.2794, -170.7004 +Palikir, 6.9248, 158.1614 +Panama City, 8.9824, -79.5199 +Papeete, -17.5350, -149.5699 +Paramaribo, 5.8520, -55.2038 +Paris, 48.8566, 2.3522 +Philipsburg, 18.0255, -63.0450 +Phnom Penh, 11.5564, 104.9282 +Plymouth, 16.7056, -62.2126 +Podgorica, 42.4304, 19.2594 +Port Louis, -20.1619, 57.4989 +Port Moresby, -9.4438, 147.1803 +Port Vila, -17.7416, 168.3213 +Port-au-Prince, 18.5944, -72.3074 +Port of Spain, 10.6596, -61.4789 +Porto-Novo, 6.4968, 2.6283 +Prague, 50.0755, 14.4378 +Praia, 14.9195, -23.5087 +Pretoria, -25.7463, 28.1876 +Pristina, 42.6629, 21.1655 +Pyongyang, 39.0392, 125.7625 +Quito, -0.1807, -78.4678 +Rabat, 33.9693, -6.9275 +Reykjavik, 64.1466, -21.9426 +Riga, 56.9496, 24.1052 +Riyadh, 24.7136, 46.6753 +Road Town, 18.4207, -64.6399 +Rome, 41.9028, 12.4964 +Roseau, 15.3092, -61.3794 +Saipan, 15.1833, 145.7500 +San José, 9.9281, -84.0907 +San Juan, 18.4655, -66.1057 +San Marino, 43.9424, 12.4578 +San Salvador, 13.6929, -89.2182 +Sana'a, 15.3694, 44.1910 +Santiago, -33.4489, -70.6693 +Santo Domingo, 18.4861, -69.9312 +Sarajevo, 43.8564, 18.4131 +Seoul, 37.5665, 126.9780 +Singapore, 1.3521, 103.8198 +Skopje, 41.9973, 21.4279 +Sofia, 42.6975, 23.3241 +Sri Jayawardenepura Kotte, 6.8928, 79.9277 +St. George's, 12.0561, -61.7485 +St. Helier, 49.1839, -2.1064 +St. John's, 17.1171, -61.8456 +St. Peter Port, 49.4599, -2.5352 +Stanley, -51.7020, -57.8517 +Stockholm, 59.3293, 18.0686 +Sucre, -19.0421, -65.2559 +Sukhumi, 43.0004, 41.0234 +Suva, -18.1416, 178.4419 +Taipei, 25.0330, 121.5654 +Tallinn, 59.4370, 24.7536 +Tarawa, 1.4170, 173.0000 +Tashkent, 41.2995, 69.2401 +Tbilisi, 41.7151, 44.8271 +Tegucigalpa, 14.0818, -87.2068 +Tehran, 35.6892, 51.3890 +Thimphu, 27.4728, 89.6390 +Tirana, 41.3275, 19.8187 +Tokyo, 35.6762, 139.6503 +Tripoli, 32.8867, 13.1910 +Tunis, 36.8065, 10.1815 +Ulaanbaatar, 47.8864, 106.9057 +Vaduz, 47.1410, 9.5215 +Valletta, 35.9042, 14.5189 +Vatican City, 41.9029, 12.4534 +Victoria, -4.6182, 55.4515 +Vienna, 48.2082, 16.3738 +Vientiane, 17.9757, 102.6331 +Vilnius, 54.6872, 25.2797 +Warsaw, 52.2297, 21.0122 +Washington D.C., 38.9072, -77.0369 +Wellington, -41.2865, 174.7762 +West Island, -12.1880, 96.8292 +Willemstad, 12.1091, -68.9319 +Windhoek, -22.5749, 17.0805 +Yamoussoukro, 6.8276, -5.2893 +Yaoundé, 3.8480, 11.5021 +Yaren, -0.5467, 166.9209 +Yerevan, 40.1872, 44.5152 +Zagreb, 45.8150, 15.9819 \ No newline at end of file diff --git a/example/vectorsearch-cities/main.py b/example/vectorsearch-cities/main.py new file mode 100644 index 0000000..4062813 --- /dev/null +++ b/example/vectorsearch-cities/main.py @@ -0,0 +1,122 @@ +from cmd import Cmd +from objectbox import Entity, Float32Vector, HnswIndex, Id, Store, String +import time +import csv +import os + + +@Entity() +class City: + id = Id() + name = String() + location = Float32Vector(index=HnswIndex(dimensions=2)) + + +def list_cities(cities): + print("{:3s} {:25s} {:>9s} {:>9s}".format("ID", "Name", "Latitude", "Longitude")) + for city in cities: + print("{:3d} {:25s} {:>9.2f} {:>9.2f}".format( + city.id, city.name, city.location[0], city.location[1])) + + +def list_cities_with_scores(city_score_tuples): + print("{:3s} {:25s} {:>9s} {:>9s} {:>5s}".format("ID", "Name", "Latitude", "Longitude", "Score")) + for (city, score) in city_score_tuples: + print("{:3d} {:25s} {:>9.2f} {:>9.2f} {:>5.2f}".format( + city.id, city.name, city.location[0], city.location[1], score)) + + +class VectorSearchCitiesCmd(Cmd): + prompt = "> " + + def __init__(self, *args): + Cmd.__init__(self, *args) + dbdir = "cities-db" + new_db = not os.path.exists(dbdir) + self._store = Store(directory=dbdir) + self._box = self._store.box(City) + if new_db: + with open(os.path.join(os.path.dirname(__file__), 'cities.csv')) as f: + r = csv.reader(f) + cities = [] + for row in r: + city = City() + city.name = row[0] + city.location = [row[1], row[2]] + cities.append(city) + self._box.put(*cities) + + def do_ls(self, name: str = ""): + """list all cities or starting with \nusage: ls []""" + qb = self._box.query(City.name.starts_with(name)) + query = qb.build() + list_cities(query.find()) + + def do_city_neighbors(self, args: str): + """find (default: 5) next neighbors to city \nusage: city_neighbors [,]""" + try: + args = args.split(',') + if len(args) > 2: + raise ValueError() + city = args[0] + if len(city) == 0: + raise ValueError() + num = 5 + if len(args) == 2: + num = int(args[1]) + qb = self._box.query(City.name.equals(city)) + query = qb.build() + cities = query.find() + if len(cities) == 1: + location = cities[0].location + # +1 for the city + qb = self._box.query( + City.location.nearest_neighbor(location, num + 1) & City.name.not_equals(city) + ) + neighbors = qb.build().find_with_scores() + list_cities_with_scores(neighbors) + else: + print(f"no city found named '{city}'") + except ValueError: + print("usage: city_neighbors [,]") + + def do_neighbors(self, args): + """find neighbors next to geo-coord .\nusage: neighbors ,,""" + try: + args = args.split(',') + if len(args) != 3: + raise ValueError() + num = int(args[0]) + geocoord = [float(args[1]), float(args[2])] + qb = self._box.query( + City.location.nearest_neighbor(geocoord, num) + ) + neighbors = qb.build().find_with_scores() + list_cities_with_scores(neighbors) + except ValueError: + print("usage: neighbors ,,") + + def do_add(self, args: str): + """add new location\nusage: add ,,""" + try: + args = args.split(',') + if len(args) != 3: + raise ValueError() + name = str(args[0]) + lat = float(args[1]) + long = float(args[2]) + city = City() + city.name = name + city.location = [lat, long] + self._box.put(city) + except ValueError: + print("usage: add ,,") + + def do_exit(self, _): + """close the program""" + raise SystemExit() + + +if __name__ == '__main__': + app = VectorSearchCitiesCmd() + app.cmdloop('Welcome to the ObjectBox vectorsearch-cities example. Type help or ? for a list of commands.') diff --git a/example/vectorsearch-cities/objectbox-model.json b/example/vectorsearch-cities/objectbox-model.json new file mode 100644 index 0000000..5d215d1 --- /dev/null +++ b/example/vectorsearch-cities/objectbox-model.json @@ -0,0 +1,36 @@ +{ + "_note1": "KEEP THIS FILE! Check it into a version control system (VCS) like git.", + "_note2": "ObjectBox manages crucial IDs for your object model. See docs for details.", + "_note3": "If you have VCS merge conflicts, you must resolve them according to ObjectBox docs.", + "modelVersionParserMinimum": 5, + "entities": [ + { + "id": "1:1326033324603613162", + "name": "City", + "lastPropertyId": "3:3620267477682371232", + "properties": [ + { + "id": "1:2574904406208562760", + "name": "id", + "type": 6, + "flags": 1 + }, + { + "id": "2:17672653531649098", + "name": "name", + "type": 9, + "flags": 0 + }, + { + "id": "3:3620267477682371232", + "name": "location", + "type": 28, + "flags": 8, + "indexId": "1:1492488102116293126" + } + ] + } + ], + "lastEntityId": "1:1326033324603613162", + "lastIndexId": "1:1492488102116293126" +} \ No newline at end of file diff --git a/objectbox/__init__.py b/objectbox/__init__.py index b08cfed..da4601d 100644 --- a/objectbox/__init__.py +++ b/objectbox/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 ObjectBox Ltd. All rights reserved. +# Copyright 2019-2024 ObjectBox Ltd. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,23 +15,59 @@ from objectbox.box import Box from objectbox.builder import Builder -from objectbox.model import Model +from objectbox.model import Model, Entity, Id, String, Index, Bool, Int8, Int16, Int32, Int64, Float32, Float64, Bytes, BoolVector, Int8Vector, Int16Vector, Int32Vector, Int64Vector, Float32Vector, Float64Vector, CharVector, BoolList, Int8List, Int16List, Int32List, Int64List, Float32List, Float64List, CharList, Date, DateNano, Flex, HnswIndex, VectorDistanceType +from objectbox.store import Store from objectbox.objectbox import ObjectBox -from objectbox.c import NotFoundException, version_core +from objectbox.c import NotFoundException, version_core, DebugFlags from objectbox.version import Version __all__ = [ 'Box', 'Builder', 'Model', + 'Entity', + 'Id', + 'Bool', + 'Int8', + 'Int16', + 'Int32', + 'Int64', + 'Float32', + 'Float64', + 'Bytes', + 'String', + 'BoolVector', + 'Int8Vector', + 'Int16Vector', + 'Int32Vector', + 'Int64Vector', + 'Float32Vector', + 'Float64Vector', + 'CharVector', + 'BoolList', + 'Int8List', + 'Int16List', + 'Int32List', + 'Int64List', + 'Float32List', + 'Float64List', + 'CharList', + 'Date', + 'DateNano', + 'Flex', + 'Index', + 'HnswIndex', + 'VectorDistanceType', + 'Store', 'ObjectBox', 'NotFoundException', 'version', 'version_info', + 'DebugFlags' ] # Python binding version -version = Version(0, 4, 0) +version = Version(4, 0, 0) def version_info(): diff --git a/objectbox/box.py b/objectbox/box.py index 9ca0ab4..cc74038 100644 --- a/objectbox/box.py +++ b/objectbox/box.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 ObjectBox Ltd. All rights reserved. +# Copyright 2019-2024 ObjectBox Ltd. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,18 +14,20 @@ from objectbox.model.entity import _Entity -from objectbox.objectbox import ObjectBox +from objectbox.store import Store +from objectbox.query_builder import QueryBuilder +from objectbox.condition import QueryCondition from objectbox.c import * class Box: - def __init__(self, ob: ObjectBox, entity: _Entity): + def __init__(self, store: Store, entity: _Entity): if not isinstance(entity, _Entity): raise Exception("Given type is not an Entity") - self._ob = ob + self._store = store self._entity = entity - self._c_box = obx_box(ob._c_store, entity.id) + self._c_box = obx_box(store._c_store, entity._id) def is_empty(self) -> bool: is_empty = ctypes.c_bool() @@ -48,16 +50,16 @@ def put(self, *objects): return self._put_one(objects[0]) def _put_one(self, obj) -> int: - id = object_id = self._entity.get_object_id(obj) + id = object_id = self._entity._get_object_id(obj) if not id: id = obx_box_id_for_put(self._c_box, 0) - data = self._entity.marshal(obj, id) + data = self._entity._marshal(obj, id) obx_box_put(self._c_box, id, bytes(data), len(data)) if id != object_id: - self._entity.set_object_id(obj, id) + self._entity._set_object_id(obj, id) return id @@ -66,7 +68,7 @@ def _put_many(self, objects) -> None: new = {} ids = {} for k in range(len(objects)): - id = self._entity.get_object_id(objects[k]) + id = self._entity._get_object_id(objects[k]) if not id: new[k] = 0 ids[k] = id @@ -88,7 +90,7 @@ def _put_many(self, objects) -> None: # we need to keep the data around until put_many is executed because obx_bytes_array_set doesn't do a copy data = {} for k in range(len(objects)): - data[k] = bytes(self._entity.marshal(objects[k], ids[k])) + data[k] = bytes(self._entity._marshal(objects[k], ids[k])) key = ctypes.c_size_t(k) # OBX_bytes_array.data[k] = data @@ -104,20 +106,23 @@ def _put_many(self, objects) -> None: # assign new IDs on the object for k in new.keys(): - self._entity.set_object_id(objects[k], ids[k]) + self._entity._set_object_id(objects[k], ids[k]) def get(self, id: int): - with self._ob.read_tx(): + with self._store.read_tx(): c_data = ctypes.c_void_p() c_size = ctypes.c_size_t() - obx_box_get(self._c_box, id, ctypes.byref( - c_data), ctypes.byref(c_size)) - + code : obx_err = obx_box_get(self._c_box, id, ctypes.byref( + c_data), ctypes.byref(c_size)) + if code == 404: + return None + elif code != 0: + raise CoreException(code) data = c_voidp_as_bytes(c_data, c_size.value) - return self._entity.unmarshal(data) + return self._entity._unmarshal(data) def get_all(self) -> list: - with self._ob.read_tx(): + with self._store.read_tx(): # OBX_bytes_array* c_bytes_array_p = obx_box_get_all(self._c_box) @@ -130,20 +135,38 @@ def get_all(self) -> list: # OBX_bytes c_bytes = c_bytes_array.data[i] data = c_voidp_as_bytes(c_bytes.data, c_bytes.size) - result.append(self._entity.unmarshal(data)) + result.append(self._entity._unmarshal(data)) return result finally: obx_bytes_array_free(c_bytes_array_p) - def remove(self, id_or_object): - if isinstance(id_or_object, self._entity.cls): - id = self._entity.get_object_id(id_or_object) + def remove(self, id_or_object) -> bool: + if isinstance(id_or_object, self._entity._user_type): + id = self._entity._get_object_id(id_or_object) else: id = id_or_object - obx_box_remove(self._c_box, id) + code : obx_err = obx_box_remove(self._c_box, id) + if code == 404: + return False + elif code != 0: + raise CoreException(code) + return True def remove_all(self) -> int: count = ctypes.c_uint64() obx_box_remove_all(self._c_box, ctypes.byref(count)) return int(count.value) + + def query(self, condition: Optional[QueryCondition] = None) -> QueryBuilder: + """ Creates a QueryBuilder for the Entity that is managed by the Box. + + :param condition: + If given, applies the given high-level condition to the new QueryBuilder object. + Useful for a user-friendly API design; for example: + ``box.query(name_property.equals("Johnny")).build()`` + """ + qb = QueryBuilder(self._store, self) + if condition is not None: + condition.apply(qb) + return qb diff --git a/objectbox/builder.py b/objectbox/builder.py index 438b564..f6176cd 100644 --- a/objectbox/builder.py +++ b/objectbox/builder.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 ObjectBox Ltd. All rights reserved. +# Copyright 2019-2024 ObjectBox Ltd. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,34 +15,27 @@ from objectbox.c import * from objectbox.model import Model -from objectbox.objectbox import ObjectBox - +from objectbox.store import Store +from objectbox.store_options import StoreOptions +from warnings import warn class Builder: def __init__(self): - self._model = Model() - self._directory = '' + """This throws a deprecation warning on initialization.""" + warn(f'Using {self.__class__.__name__} is deprecated, please use Store(model=, directory= ...) from objectbox.store.', DeprecationWarning, stacklevel=2) + self._kwargs = { } def directory(self, path: str) -> 'Builder': - self._directory = path + self._kwargs['directory'] = path return self - def model(self, model: Model) -> 'Builder': - self._model = model - self._model._finish() + def max_db_size_in_kb(self, size_in_kb: int) -> 'Builder': + self._kwargs['max_db_size_in_kb'] = size_in_kb return self - def build(self) -> 'ObjectBox': - c_options = obx_opt() - - try: - if len(self._directory) > 0: - obx_opt_directory(c_options, c_str(self._directory)) - - obx_opt_model(c_options, self._model._c_model) - except CoreException: - obx_opt_free(c_options) - raise + def model(self, model: Model) -> 'Builder': + self._kwargs['model'] = model + return self - c_store = obx_store_open(c_options) - return ObjectBox(c_store) + def build(self) -> 'Store': + return Store(**self._kwargs) \ No newline at end of file diff --git a/objectbox/c.py b/objectbox/c.py index 339b530..7c005bd 100644 --- a/objectbox/c.py +++ b/objectbox/c.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 ObjectBox Ltd. All rights reserved. +# Copyright 2019-2024 ObjectBox Ltd. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,6 +17,9 @@ import os import platform from objectbox.version import Version +from typing import * +import numpy as np +from enum import IntEnum # This file contains C-API bindings based on lib/objectbox.h, linking to the 'objectbox' shared library. # The bindings are implementing using ctypes, see https://docs.python.org/dev/library/ctypes.html for introduction. @@ -24,7 +27,7 @@ # Version of the library used by the binding. This version is checked at runtime to ensure binary compatibility. # Don't forget to update download-c-lib.py when upgrading to a newer version. -required_version = "0.14.0" +required_version = "4.0.0" def shlib_name(library: str) -> str: @@ -42,7 +45,8 @@ def shlib_name(library: str) -> str: # initialize the C library lib_path = os.path.dirname(os.path.realpath(__file__)) lib_path = os.path.join(lib_path, 'lib', - platform.machine() if platform.system() != 'Darwin' else 'macos-universal', shlib_name('objectbox')) + platform.machine() if platform.system() != 'Darwin' else 'macos-universal', + shlib_name('objectbox')) C = ctypes.CDLL(lib_path) # load the core library version @@ -66,12 +70,31 @@ def shlib_name(library: str) -> str: obx_id = ctypes.c_uint64 obx_qb_cond = ctypes.c_int +obx_qb_cond_p = ctypes.POINTER(obx_qb_cond) + # enums OBXPropertyType = ctypes.c_int OBXPropertyFlags = ctypes.c_int OBXDebugFlags = ctypes.c_int OBXPutMode = ctypes.c_int +OBXPutPaddingMode = ctypes.c_int OBXOrderFlags = ctypes.c_int +OBXHnswFlags = ctypes.c_int +OBXVectorDistanceType = ctypes.c_int +OBXValidateOnOpenPagesFlags = ctypes.c_int +OBXValidateOnOpenKvFlags = ctypes.c_int +OBXBackupRestoreFlags = ctypes.c_int + +class DebugFlags(IntEnum): + NONE = 0, + LOG_TRANSACTIONS_READ = 1, + LOG_TRANSACTIONS_WRITE = 2, + LOG_QUERIES = 3, + LOG_QUERY_PARAMETERS = 8, + LOG_ASYNC_QUEUE = 16, + LOG_CACHE_HITS = 32, + LOG_CACHE_ALL = 64, + LOG_TREE = 128 class OBX_model(ctypes.Structure): @@ -115,6 +138,27 @@ class OBX_bytes_array(ctypes.Structure): OBX_bytes_array_p = ctypes.POINTER(OBX_bytes_array) +class OBX_bytes_score(ctypes.Structure): + _fields_ = [ + ('data', ctypes.c_void_p), + ('size', ctypes.c_size_t), + ('score', ctypes.c_double), + ] + + +OBX_bytes_score_p = ctypes.POINTER(OBX_bytes_score) + + +class OBX_bytes_score_array(ctypes.Structure): + _fields_ = [ + ('bytes_scores', OBX_bytes_score_p), + ('count', ctypes.c_size_t), + ] + + +OBX_bytes_score_array_p = ctypes.POINTER(OBX_bytes_score_array) + + class OBX_id_array(ctypes.Structure): _fields_ = [ ('ids', ctypes.POINTER(obx_id)), @@ -125,6 +169,26 @@ class OBX_id_array(ctypes.Structure): OBX_id_array_p = ctypes.POINTER(OBX_id_array) +class OBX_id_score(ctypes.Structure): + _fields_ = [ + ('id', obx_id), + ('score', ctypes.c_double) + ] + + +OBX_id_score_p = ctypes.POINTER(OBX_id_score) + + +class OBX_id_score_array(ctypes.Structure): + _fields_ = [ + ('ids_scores', OBX_id_score_p), + ('count', ctypes.c_size_t) + ] + + +OBX_id_score_array_p = ctypes.POINTER(OBX_id_score_array) + + class OBX_txn(ctypes.Structure): pass @@ -193,25 +257,35 @@ class CoreException(Exception): 10502: "FILE_CORRUPT" } - def __init__(self, code): + def __init__(self, code: int): self.code = code self.message = py_str(C.obx_last_error_message()) name = self.codes[code] if code in self.codes else "n/a" - super(CoreException, self).__init__( - "%d (%s) - %s" % (code, name, self.message)) + super(CoreException, self).__init__("%d (%s) - %s" % (code, name, self.message)) + + @staticmethod + def last(): + """ Creates a CoreException of the last error that was generated in core. """ + return CoreException(C.obx_last_error()) class NotFoundException(Exception): pass -# assert the the returned obx_err is empty def check_obx_err(code: obx_err, func, args) -> obx_err: + """ Raises an exception if obx_err is not successful. """ if code == 404: raise NotFoundException() elif code != 0: raise CoreException(code) + return code + +def check_obx_qb_cond(code: obx_qb_cond, func, args) -> obx_qb_cond: + """ Raises an exception if obx_qb_cond is not successful. """ + if code == 0: + raise CoreException(code) return code @@ -222,21 +296,47 @@ def check_result(result, func, args): return result -# creates a global function "name" with the given restype & argtypes, calling C function with the same name -def fn(name: str, restype: type, argtypes): +# creates a global function "name" with the given restype & argtypes, calling C function with the same name. +# restype is used for error checking: if not None, check_result will throw an exception if the result is empty. +def c_fn(name: str, restype: Optional[type], argtypes): func = C.__getattr__(name) - func.argtypes = argtypes func.restype = restype - if restype is obx_err: - func.errcheck = check_obx_err - elif restype is not None: + if restype is not None: func.errcheck = check_result return func +# creates a global function "name" with the given restype & argtypes, calling C function with the same name. +# no error checking is done on restype as this is defered to higher-level functions. +def c_fn_nocheck(name: str, restype: type, argtypes): + func = C.__getattr__(name) + func.argtypes = argtypes + func.restype = restype + return func + + +# like c_fn, but for functions returning obx_err +def c_fn_rc(name: str, argtypes): + """ Like c_fn, but for functions returning obx_err (checks obx_err validity). """ + func = C.__getattr__(name) + func.argtypes = argtypes + func.restype = obx_err + func.errcheck = check_obx_err + return func + + +def c_fn_qb_cond(name: str, argtypes): + """ Like c_fn, but for functions returning obx_qb_cond (checks obx_qb_cond validity). """ + func = C.__getattr__(name) + func.argtypes = argtypes + func.restype = obx_qb_cond + func.errcheck = check_obx_qb_cond + return func + + def py_str(ptr: ctypes.c_char_p) -> str: return ctypes.c_char_p(ptr).value.decode("utf-8") @@ -255,133 +355,565 @@ def c_voidp_as_bytes(voidp, size): return memoryview(ctypes.cast(voidp, ctypes.POINTER(ctypes.c_ubyte * size))[0]).tobytes() +def c_array(py_list: Union[List[Any], np.ndarray], c_type): + """ Converts the given python list or ndarray into a C array of c_type. """ + if isinstance(py_list, np.ndarray): + if py_list.ndim != 1: + raise Exception(f"ndarray is expected to be 1-dimensional. Input shape: {py_list.shape}") + return py_list.ctypes.data_as(ctypes.POINTER(c_type)) + elif isinstance(py_list, list): + return (c_type * len(py_list))(*py_list) + else: + raise Exception(f"Unsupported Python list type: {type(py_list)}") + + +def c_array_pointer(py_list: Union[List[Any], np.ndarray], c_type): + """ Converts the given python list or ndarray into a C array of c_type. Returns its pointer type. """ + return ctypes.cast(c_array(py_list, c_type), ctypes.POINTER(c_type)) + + +# OBX_C_API float obx_vector_distance_float32(OBXVectorDistanceType type, const float* vector1, const float* vector2, size_t dimension); +obx_vector_distance_float32 = c_fn("obx_vector_distance_float32", ctypes.c_float, [OBXVectorDistanceType, ctypes.POINTER(ctypes.c_float), ctypes.POINTER(ctypes.c_float), ctypes.c_size_t]) + +# OBX_C_API float obx_vector_distance_to_relevance(OBXVectorDistanceType type, float distance); +obx_vector_distance_to_relevance = c_fn("obx_vector_distance_to_relevance", ctypes.c_float, [OBXVectorDistanceType, ctypes.c_float]) + # OBX_model* (void); -obx_model = fn('obx_model', OBX_model_p, []) +obx_model = c_fn('obx_model', OBX_model_p, []) # obx_err (OBX_model* model, const char* name, obx_schema_id entity_id, obx_uid entity_uid); -obx_model_entity = fn('obx_model_entity', obx_err, [ - OBX_model_p, ctypes.c_char_p, obx_schema_id, obx_uid]) +obx_model_entity = c_fn_rc('obx_model_entity', [ + OBX_model_p, ctypes.c_char_p, obx_schema_id, obx_uid]) # obx_err (OBX_model* model, const char* name, OBXPropertyType type, obx_schema_id property_id, obx_uid property_uid); -obx_model_property = fn('obx_model_property', obx_err, - [OBX_model_p, ctypes.c_char_p, OBXPropertyType, obx_schema_id, obx_uid]) +obx_model_property = c_fn_rc('obx_model_property', + [OBX_model_p, ctypes.c_char_p, OBXPropertyType, obx_schema_id, obx_uid]) # obx_err (OBX_model* model, OBXPropertyFlags flags); -obx_model_property_flags = fn('obx_model_property_flags', obx_err, [ - OBX_model_p, OBXPropertyFlags]) +obx_model_property_flags = c_fn_rc('obx_model_property_flags', [OBX_model_p, OBXPropertyFlags]) + +# obx_err obx_model_property_index_id(OBX_model* model, obx_schema_id index_id, obx_uid index_uid) +obx_model_property_index_id = c_fn_rc('obx_model_property_index_id', [OBX_model_p, obx_schema_id, obx_uid]) + +# obx_err obx_model_property_index_hnsw_dimensions(OBX_model* model, size_t value) +obx_model_property_index_hnsw_dimensions = \ + c_fn_rc('obx_model_property_index_hnsw_dimensions', [OBX_model_p, ctypes.c_size_t]) + +# obx_err obx_model_property_index_hnsw_neighbors_per_node(OBX_model* model, uint32_t value) +obx_model_property_index_hnsw_neighbors_per_node = \ + c_fn_rc('obx_model_property_index_hnsw_neighbors_per_node', [OBX_model_p, ctypes.c_uint32]) + +# obx_err obx_model_property_index_hnsw_indexing_search_count(OBX_model* model, uint32_t value) +obx_model_property_index_hnsw_indexing_search_count = \ + c_fn_rc('obx_model_property_index_hnsw_indexing_search_count', [OBX_model_p, ctypes.c_uint32]) + +# obx_err obx_model_property_index_hnsw_flags(OBX_model* model, OBXHnswFlags value) +obx_model_property_index_hnsw_flags = \ + c_fn_rc('obx_model_property_index_hnsw_flags', [OBX_model_p, OBXHnswFlags]) + +# obx_err obx_model_property_index_hnsw_distance_type(OBX_model* model, OBXVectorDistanceType value) +obx_model_property_index_hnsw_distance_type = c_fn_rc('obx_model_property_index_hnsw_distance_type', [OBX_model_p, OBXVectorDistanceType]) + +# obx_err obx_model_property_index_hnsw_reparation_backlink_probability(OBX_model* model, float value) +obx_model_property_index_hnsw_reparation_backlink_probability = \ + c_fn_rc('obx_model_property_index_hnsw_reparation_backlink_probability', [OBX_model_p, ctypes.c_float]) + +# obx_err obx_model_property_index_hnsw_vector_cache_hint_size_kb(OBX_model* model, size_t value) +obx_model_property_index_hnsw_vector_cache_hint_size_kb = \ + c_fn_rc('obx_model_property_index_hnsw_vector_cache_hint_size_kb', [OBX_model_p, ctypes.c_size_t]) # obx_err (OBX_model*, obx_schema_id entity_id, obx_uid entity_uid); -obx_model_last_entity_id = fn('obx_model_last_entity_id', None, [ - OBX_model_p, obx_schema_id, obx_uid]) +obx_model_last_entity_id = c_fn('obx_model_last_entity_id', None, [ + OBX_model_p, obx_schema_id, obx_uid]) # obx_err (OBX_model* model, obx_schema_id index_id, obx_uid index_uid); -obx_model_last_index_id = fn('obx_model_last_index_id', None, [ - OBX_model_p, obx_schema_id, obx_uid]) +obx_model_last_index_id = c_fn('obx_model_last_index_id', None, [ + OBX_model_p, obx_schema_id, obx_uid]) # obx_err (OBX_model* model, obx_schema_id relation_id, obx_uid relation_uid); -obx_model_last_relation_id = fn('obx_model_last_relation_id', None, [ - OBX_model_p, obx_schema_id, obx_uid]) +obx_model_last_relation_id = c_fn('obx_model_last_relation_id', None, [ + OBX_model_p, obx_schema_id, obx_uid]) # obx_err (OBX_model* model, obx_schema_id property_id, obx_uid property_uid); -obx_model_entity_last_property_id = fn('obx_model_entity_last_property_id', obx_err, - [OBX_model_p, obx_schema_id, obx_uid]) +obx_model_entity_last_property_id = c_fn_rc('obx_model_entity_last_property_id', + [OBX_model_p, obx_schema_id, obx_uid]) # OBX_store_options* (); -obx_opt = fn('obx_opt', OBX_store_options_p, []) +obx_opt = c_fn('obx_opt', OBX_store_options_p, []) + +# OBX_C_API obx_err obx_opt_directory(OBX_store_options* opt, const char* dir); +obx_opt_directory = c_fn_rc('obx_opt_directory', [OBX_store_options_p, ctypes.c_char_p]) + +# OBX_C_API void obx_opt_max_db_size_in_kb(OBX_store_options* opt, uint64_t size_in_kb); +obx_opt_max_db_size_in_kb = c_fn('obx_opt_max_db_size_in_kb', None, [OBX_store_options_p, ctypes.c_uint64]) + +# OBX_C_API void obx_opt_max_data_size_in_kb(OBX_store_options* opt, uint64_t size_in_kb); +obx_opt_max_data_size_in_kb = c_fn('obx_opt_max_data_size_in_kb', None, [OBX_store_options_p, ctypes.c_uint64]) + +# OBX_C_API void obx_opt_file_mode(OBX_store_options* opt, unsigned int file_mode); +obx_opt_file_mode = c_fn('obx_opt_file_mode', None, [OBX_store_options_p, ctypes.c_uint32]) + +# OBX_C_API void obx_opt_max_readers(OBX_store_options* opt, unsigned int max_readers); +obx_opt_max_readers = c_fn('obx_opt_max_readers', None, [OBX_store_options_p, ctypes.c_uint32]) + +# OBX_C_API void obx_opt_no_reader_thread_locals(OBX_store_options* opt, bool flag); +obx_opt_no_reader_thread_locals = c_fn('obx_opt_no_reader_thread_locals', None, [OBX_store_options_p, ctypes.c_bool]) + +# OBX_C_API obx_err obx_opt_model(OBX_store_options* opt, OBX_model* model); +obx_opt_model = c_fn_rc('obx_opt_model', [OBX_store_options_p, OBX_model_p]) -# obx_err (OBX_store_options* opt, const char* dir); -obx_opt_directory = fn('obx_opt_directory', obx_err, [ - OBX_store_options_p, ctypes.c_char_p]) +# OBX_C_API obx_err obx_opt_model_bytes(OBX_store_options* opt, const void* bytes, size_t size); +obx_opt_model_bytes = c_fn_rc('obx_opt_model_bytes', [OBX_store_options_p, ctypes.c_void_p, ctypes.c_size_t]) -# void (OBX_store_options* opt, size_t size_in_kb); -obx_opt_max_db_size_in_kb = fn('obx_opt_max_db_size_in_kb', None, [ - OBX_store_options_p, ctypes.c_size_t]) +# OBX_C_API obx_err obx_opt_model_bytes_direct(OBX_store_options* opt, const void* bytes, size_t size); +obx_opt_model_bytes_direct = c_fn_rc('obx_opt_model_bytes_direct', [OBX_store_options_p, ctypes.c_void_p, ctypes.c_size_t]) -# void (OBX_store_options* opt, int file_mode); -obx_opt_file_mode = fn('obx_opt_file_mode', None, [ - OBX_store_options_p, ctypes.c_uint]) +# OBX_C_API void obx_opt_validate_on_open_pages(OBX_store_options* opt, size_t page_limit, uint32_t flags); +obx_opt_validate_on_open_pages = c_fn('obx_opt_validate_on_open_pages', None, [OBX_store_options_p, ctypes.c_size_t, OBXValidateOnOpenPagesFlags]) -# void (OBX_store_options* opt, int max_readers); -obx_opt_max_readers = fn('obx_opt_max_readers', None, [ - OBX_store_options_p, ctypes.c_uint]) +# OBX_C_API void obx_opt_validate_on_open_kv(OBX_store_options* opt, uint32_t flags); +obx_opt_validate_on_open_kv = c_fn('obx_opt_validate_on_open_kv', None, [OBX_store_options_p, OBXValidateOnOpenKvFlags]) -# obx_err (OBX_store_options* opt, OBX_model* model); -obx_opt_model = fn('obx_opt_model', obx_err, [ - OBX_store_options_p, OBX_model_p]) +# OBX_C_API void obx_opt_put_padding_mode(OBX_store_options* opt, OBXPutPaddingMode mode); +obx_opt_put_padding_mode = c_fn('obx_opt_put_padding_mode', None, [OBX_store_options_p, OBXPutPaddingMode]) -# void (OBX_store_options* opt); -obx_opt_free = fn('obx_opt_free', None, [OBX_store_options_p]) +# OBX_C_API void obx_opt_read_schema(OBX_store_options* opt, bool value); +obx_opt_read_schema = c_fn('obx_opt_read_schema', None, [OBX_store_options_p, ctypes.c_bool]) + +# OBX_C_API void obx_opt_use_previous_commit(OBX_store_options* opt, bool value); +obx_opt_use_previous_commit = c_fn('obx_opt_use_previous_commit', None, [OBX_store_options_p, ctypes.c_bool]) + +# OBX_C_API void obx_opt_read_only(OBX_store_options* opt, bool value); +obx_opt_read_only = c_fn('obx_opt_read_only', None, [OBX_store_options_p, ctypes.c_bool]) + +# OBX_C_API void obx_opt_debug_flags(OBX_store_options* opt, uint32_t flags); +obx_opt_debug_flags = c_fn('obx_opt_debug_flags', None, [OBX_store_options_p, OBXDebugFlags]) + +# OBX_C_API void obx_opt_add_debug_flags(OBX_store_options* opt, uint32_t flags); +obx_opt_add_debug_flags = c_fn('obx_opt_add_debug_flags', None, [OBX_store_options_p, ctypes.c_uint32]) + +# OBX_C_API void obx_opt_async_max_queue_length(OBX_store_options* opt, size_t value); +obx_opt_async_max_queue_length = c_fn('obx_opt_async_max_queue_length', None, [OBX_store_options_p, ctypes.c_size_t]) + +# OBX_C_API void obx_opt_async_throttle_at_queue_length(OBX_store_options* opt, size_t value); +obx_opt_async_throttle_at_queue_length = c_fn('obx_opt_async_throttle_at_queue_length', None, [OBX_store_options_p, ctypes.c_size_t]) + +# OBX_C_API void obx_opt_async_throttle_micros(OBX_store_options* opt, uint32_t value); +obx_opt_async_throttle_micros = c_fn('obx_opt_async_throttle_micros', None, [OBX_store_options_p, ctypes.c_uint32]) + +# OBX_C_API void obx_opt_async_max_in_tx_duration(OBX_store_options* opt, uint32_t micros); +obx_opt_async_max_in_tx_duration = c_fn('obx_opt_async_max_in_tx_duration', None, [OBX_store_options_p, ctypes.c_uint32]) + +# OBX_C_API void obx_opt_async_max_in_tx_operations(OBX_store_options* opt, uint32_t value); +obx_opt_async_max_in_tx_operations = c_fn('obx_opt_async_max_in_tx_operations', None, [OBX_store_options_p, ctypes.c_uint32]) + +# OBX_C_API void obx_opt_async_pre_txn_delay(OBX_store_options* opt, uint32_t delay_micros); +obx_opt_async_pre_txn_delay = c_fn('obx_opt_async_pre_txn_delay', None, [OBX_store_options_p, ctypes.c_uint32]) + +# OBX_C_API void obx_opt_async_pre_txn_delay4(OBX_store_options* opt, uint32_t delay_micros, uint32_t delay2_micros, size_t min_queue_length_for_delay2); +obx_opt_async_pre_txn_delay4 = c_fn('obx_opt_async_pre_txn_delay4', None, [OBX_store_options_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_size_t]) + +# OBX_C_API void obx_opt_async_post_txn_delay(OBX_store_options* opt, uint32_t delay_micros); +obx_opt_async_post_txn_delay = c_fn('obx_opt_async_post_txn_delay', None, [OBX_store_options_p, ctypes.c_uint32]) + +# OBX_C_API void obx_opt_async_post_txn_delay5(OBX_store_options* opt, uint32_t delay_micros, uint32_t delay2_micros, size_t min_queue_length_for_delay2, bool subtract_processing_time); +obx_opt_async_post_txn_delay5 = c_fn('obx_opt_async_post_txn_delay5', None, [OBX_store_options_p, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_size_t, ctypes.c_bool]) + +# OBX_C_API void obx_opt_async_minor_refill_threshold(OBX_store_options* opt, size_t queue_length); +obx_opt_async_minor_refill_threshold = c_fn('obx_opt_async_minor_refill_threshold', None, [OBX_store_options_p, ctypes.c_size_t]) + +# OBX_C_API void obx_opt_async_minor_refill_max_count(OBX_store_options* opt, uint32_t value); +obx_opt_async_minor_refill_max_count = c_fn('obx_opt_async_minor_refill_max_count', None, [OBX_store_options_p, ctypes.c_uint32]) + +# OBX_C_API void obx_opt_async_max_tx_pool_size(OBX_store_options* opt, size_t value); +obx_opt_async_max_tx_pool_size = c_fn('obx_opt_async_max_tx_pool_size', None, [OBX_store_options_p, ctypes.c_size_t]) + +# OBX_C_API void obx_opt_async_object_bytes_max_cache_size(OBX_store_options* opt, uint64_t value); +obx_opt_async_object_bytes_max_cache_size = c_fn('obx_opt_async_object_bytes_max_cache_size', None, [OBX_store_options_p, ctypes.c_uint64]) + +# OBX_C_API void obx_opt_async_object_bytes_max_size_to_cache(OBX_store_options* opt, uint64_t value); +obx_opt_async_object_bytes_max_size_to_cache = c_fn('obx_opt_async_object_bytes_max_size_to_cache', None, [OBX_store_options_p, ctypes.c_uint64]) + +# OBX_C_API void obx_opt_log_callback(OBX_store_options* opt, obx_log_callback* callback, void* user_data); +# obx_opt_log_callback = c_fn('obx_opt_log_callback', None, [OBX_store_options_p, ...]) TODO + +# OBX_C_API void obx_opt_backup_restore(OBX_store_options* opt, const char* backup_file, uint32_t flags); +obx_opt_backup_restore = c_fn('obx_opt_backup_restore', None, [OBX_store_options_p, ctypes.c_char_p, OBXBackupRestoreFlags]) + +# OBX_C_API const char* obx_opt_get_directory(OBX_store_options* opt); +obx_opt_get_directory = c_fn('obx_opt_get_directory', ctypes.c_char_p, [OBX_store_options_p]) + +# OBX_C_API uint64_t obx_opt_get_max_db_size_in_kb(OBX_store_options* opt); +obx_opt_get_max_db_size_in_kb = c_fn('obx_opt_get_max_db_size_in_kb', ctypes.c_uint64, [OBX_store_options_p]) + +# OBX_C_API uint64_t obx_opt_get_max_data_size_in_kb(OBX_store_options* opt); +obx_opt_get_max_data_size_in_kb = c_fn('obx_opt_get_max_data_size_in_kb', ctypes.c_uint64, [OBX_store_options_p]) + +# OBX_C_API uint32_t obx_opt_get_debug_flags(OBX_store_options* opt); +obx_opt_get_debug_flags = c_fn('obx_opt_get_debug_flags', ctypes.c_uint32, [OBX_store_options_p]) + +# OBX_C_API void obx_opt_free(OBX_store_options* opt); +obx_opt_free = c_fn('obx_opt_free', None, []) # OBX_store* (const OBX_store_options* options); -obx_store_open = fn('obx_store_open', OBX_store_p, [OBX_store_options_p]) +obx_store_open = c_fn('obx_store_open', OBX_store_p, [OBX_store_options_p]) # obx_err (OBX_store* store); -obx_store_close = fn('obx_store_close', obx_err, [OBX_store_p]) +obx_store_close = c_fn_rc('obx_store_close', [OBX_store_p]) + +# obx_err obx_remove_db_files(const const* directory); +obx_remove_db_files = c_fn_rc('obx_remove_db_files', [ctypes.c_char_p]) # TODO provide a python wrapper # OBX_txn* (OBX_store* store); -obx_txn_write = fn('obx_txn_write', OBX_txn_p, [OBX_store_p]) +obx_txn_write = c_fn('obx_txn_write', OBX_txn_p, [OBX_store_p]) # OBX_txn* (OBX_store* store); -obx_txn_read = fn('obx_txn_read', OBX_txn_p, [OBX_store_p]) +obx_txn_read = c_fn('obx_txn_read', OBX_txn_p, [OBX_store_p]) # obx_err (OBX_txn* txn) -obx_txn_close = fn('obx_txn_close', obx_err, [OBX_txn_p]) +obx_txn_close = c_fn_rc('obx_txn_close', [OBX_txn_p]) # obx_err (OBX_txn* txn); -obx_txn_abort = fn('obx_txn_abort', obx_err, [OBX_txn_p]) +obx_txn_abort = c_fn_rc('obx_txn_abort', [OBX_txn_p]) # obx_err (OBX_txn* txn); -obx_txn_success = fn('obx_txn_success', obx_err, [OBX_txn_p]) +obx_txn_success = c_fn_rc('obx_txn_success', [OBX_txn_p]) # OBX_box* (OBX_store* store, obx_schema_id entity_id); -obx_box = fn('obx_box', OBX_box_p, [OBX_store_p, obx_schema_id]) +obx_box = c_fn('obx_box', OBX_box_p, [OBX_store_p, obx_schema_id]) # obx_err (OBX_box* box, obx_id id, const void** data, size_t* size); -obx_box_get = fn('obx_box_get', obx_err, - [OBX_box_p, obx_id, ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(ctypes.c_size_t)]) +obx_box_get = c_fn_nocheck('obx_box_get', obx_err, [ + OBX_box_p, obx_id, ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(ctypes.c_size_t)]) # OBX_bytes_array* (OBX_box* box); -obx_box_get_all = fn('obx_box_get_all', OBX_bytes_array_p, [OBX_box_p]) +obx_box_get_all = c_fn('obx_box_get_all', OBX_bytes_array_p, [OBX_box_p]) # obx_id (OBX_box* box, obx_id id_or_zero); -obx_box_id_for_put = fn('obx_box_id_for_put', obx_id, [OBX_box_p, obx_id]) +obx_box_id_for_put = c_fn('obx_box_id_for_put', obx_id, [OBX_box_p, obx_id]) # obx_err (OBX_box* box, uint64_t count, obx_id* out_first_id); -obx_box_ids_for_put = fn('obx_box_ids_for_put', obx_err, [ - OBX_box_p, ctypes.c_uint64, ctypes.POINTER(obx_id)]) +obx_box_ids_for_put = c_fn_rc('obx_box_ids_for_put', [ + OBX_box_p, ctypes.c_uint64, ctypes.POINTER(obx_id)]) # obx_err (OBX_box* box, obx_id id, const void* data, size_t size); -obx_box_put = fn('obx_box_put', obx_err, [ - OBX_box_p, obx_id, ctypes.c_void_p, ctypes.c_size_t]) +obx_box_put = c_fn_rc('obx_box_put', [OBX_box_p, obx_id, ctypes.c_void_p, ctypes.c_size_t]) # obx_err (OBX_box* box, const OBX_bytes_array* objects, const obx_id* ids, OBXPutMode mode); -obx_box_put_many = fn('obx_box_put_many', obx_err, [ - OBX_box_p, OBX_bytes_array_p, ctypes.POINTER(obx_id), OBXPutMode]) +obx_box_put_many = c_fn_rc('obx_box_put_many', [ + OBX_box_p, OBX_bytes_array_p, ctypes.POINTER(obx_id), OBXPutMode]) # obx_err (OBX_box* box, obx_id id); -obx_box_remove = fn('obx_box_remove', obx_err, [OBX_box_p, obx_id]) +obx_box_remove = c_fn_nocheck('obx_box_remove', obx_err, [OBX_box_p, obx_id]) # obx_err (OBX_box* box, uint64_t* out_count); -obx_box_remove_all = fn('obx_box_remove_all', obx_err, [ - OBX_box_p, ctypes.POINTER(ctypes.c_uint64)]) +obx_box_remove_all = c_fn_rc('obx_box_remove_all', [ + OBX_box_p, ctypes.POINTER(ctypes.c_uint64)]) # obx_err (OBX_box* box, bool* out_is_empty); -obx_box_is_empty = fn('obx_box_is_empty', obx_err, [ - OBX_box_p, ctypes.POINTER(ctypes.c_bool)]) +obx_box_is_empty = c_fn_rc('obx_box_is_empty', [ + OBX_box_p, ctypes.POINTER(ctypes.c_bool)]) # obx_err obx_box_count(OBX_box* box, uint64_t limit, uint64_t* out_count); -obx_box_count = fn('obx_box_count', obx_err, [ - OBX_box_p, ctypes.c_uint64, ctypes.POINTER(ctypes.c_uint64)]) +obx_box_count = c_fn_rc('obx_box_count', [ + OBX_box_p, ctypes.c_uint64, ctypes.POINTER(ctypes.c_uint64)]) + +# OBX_query_builder* obx_query_builder(OBX_store* store, obx_schema_id entity_id); +obx_query_builder = c_fn('obx_query_builder', OBX_query_builder_p, [OBX_store_p, obx_schema_id]) + +# OBX_C_API obx_err obx_qb_close(OBX_query_builder* builder); +obx_qb_close = c_fn_rc('obx_qb_close', [OBX_query_builder_p]) + +# OBX_C_API OBX_query* obx_query(OBX_query_builder* builder); +obx_query = c_fn('obx_query', OBX_query_p, [OBX_query_builder_p]) + +# OBX_C_API obx_err obx_qb_error_code(OBX_query_builder* builder); +obx_qb_error_code = c_fn_rc('obx_qb_error_code', [OBX_query_builder_p]) + +# OBX_C_API const char* obx_qb_error_message(OBX_query_builder* builder); +obx_qb_error_message = c_fn('obx_qb_error_message', ctypes.c_char_p, [OBX_query_builder_p]) + +# OBX_C_API obx_qb_cond obx_qb_null(OBX_query_builder* builder, obx_schema_id property_id); +obx_qb_null = c_fn('obx_qb_null', obx_qb_cond, [OBX_query_builder_p, obx_schema_id]) + +# OBX_C_API obx_qb_cond obx_qb_not_null(OBX_query_builder* builder, obx_schema_id property_id); +obx_qb_not_null = c_fn('obx_qb_not_null', obx_qb_cond, [OBX_query_builder_p, obx_schema_id]) + +# OBX_C_API obx_qb_cond obx_qb_equals_string(OBX_query_builder* builder, obx_schema_id property_id, const char* value, +# bool case_sensitive); +obx_qb_equals_string = c_fn('obx_qb_equals_string', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_char_p, ctypes.c_bool]) + +# OBX_C_API obx_qb_co nd obx_qb_not_equals_string(OBX_query_builder* builder, obx_schema_id property_id, const char* value, +# bool case_sensitive); +obx_qb_not_equals_string = c_fn('obx_qb_not_equals_string', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_char_p, ctypes.c_bool]) + +# OBX_C_API obx_qb_cond obx_qb_contains_string(OBX_query_builder* builder, obx_schema_id property_id, const char* value, +# bool case_sensitive); +obx_qb_contains_string = c_fn('obx_qb_contains_string', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_char_p, ctypes.c_bool]) + +# OBX_C_API obx_qb_cond obx_qb_contains_element_string(OBX_query_builder* builder, obx_schema_id property_id, +# const char* value, bool case_sensitive); +obx_qb_contains_element_string = c_fn('obx_qb_contains_element_string', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_char_p, ctypes.c_bool]) + +# OBX_C_API obx_qb_cond obx_qb_contains_key_value_string(OBX_query_builder* builder, obx_schema_id property_id, +# const char* key, const char* value, bool case_sensitive); +obx_qb_contains_key_value_string = c_fn('obx_qb_contains_key_value_string', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_char_p, ctypes.c_char_p, + ctypes.c_bool]) + +# OBX_C_API obx_qb_cond obx_qb_starts_with_string(OBX_query_builder* builder, obx_schema_id property_id, +# const char* value, bool case_sensitive); +obx_qb_starts_with_string = c_fn('obx_qb_starts_with_string', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_char_p, ctypes.c_bool]) + +# OBX_C_API obx_qb_cond obx_qb_ends_with_string(OBX_query_builder* builder, obx_schema_id property_id, const char* value, +# bool case_sensitive); +obx_qb_ends_with_string = c_fn('obx_qb_ends_with_string', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_char_p, ctypes.c_bool]) + +# OBX_C_API obx_qb_cond obx_qb_greater_than_string(OBX_query_builder* builder, obx_schema_id property_id, +# const char* value, bool case_sensitive); +obx_qb_greater_than_string = c_fn('obx_qb_greater_than_string', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_char_p, ctypes.c_bool]) + +# OBX_C_API obx_qb_cond obx_qb_greater_or_equal_string(OBX_query_builder* builder, obx_schema_id property_id, +# const char* value, bool case_sensitive); +obx_qb_greater_or_equal_string = c_fn('obx_qb_greater_or_equal_string', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_char_p, ctypes.c_bool]) + +# OBX_C_API obx_qb_cond obx_qb_less_than_string(OBX_query_builder* builder, obx_schema_id property_id, const char* value, +# bool case_sensitive); +obx_qb_less_than_string = c_fn('obx_qb_less_than_string', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_char_p, ctypes.c_bool]) + +# OBX_C_API obx_qb_cond obx_qb_less_or_equal_string(OBX_query_builder* builder, obx_schema_id property_id, +# const char* value, bool case_sensitive); +obx_qb_less_or_equal_string = c_fn('obx_qb_less_or_equal_string', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_char_p, ctypes.c_bool]) + +# OBX_C_API obx_qb_cond obx_qb_in_strings(OBX_query_builder* builder, obx_schema_id property_id, +# const char* const values[], size_t count, bool case_sensitive); +obx_qb_in_strings = c_fn('obx_qb_in_strings', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.POINTER(ctypes.c_char_p), ctypes.c_size_t, + ctypes.c_bool]) + +# OBX_C_API obx_qb_cond obx_qb_any_equals_string(OBX_query_builder* builder, obx_schema_id property_id, const char* value, +# bool case_sensitive); +obx_qb_any_equals_string = c_fn('obx_qb_any_equals_string', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_char_p, ctypes.c_bool]) + +# OBX_C_API obx_qb_cond obx_qb_equals_int(OBX_query_builder* builder, obx_schema_id property_id, int64_t value); +obx_qb_equals_int = c_fn('obx_qb_equals_int', obx_qb_cond, [OBX_query_builder_p, obx_schema_id, ctypes.c_int64]) + +# OBX_C_API obx_qb_cond obx_qb_not_equals_int(OBX_query_builder* builder, obx_schema_id property_id, int64_t value); +obx_qb_not_equals_int = c_fn('obx_qb_not_equals_int', obx_qb_cond, [OBX_query_builder_p, obx_schema_id, ctypes.c_int64]) + +# OBX_C_API obx_qb_cond obx_qb_greater_than_int(OBX_query_builder* builder, obx_schema_id property_id, int64_t value); +obx_qb_greater_than_int = c_fn('obx_qb_greater_than_int', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_int64]) + +# OBX_C_API obx_qb_cond obx_qb_greater_or_equal_int(OBX_query_builder* builder, obx_schema_id property_id, int64_t value); +obx_qb_greater_or_equal_int = c_fn('obx_qb_greater_or_equal_int', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_int64]) + +# OBX_C_API obx_qb_cond obx_qb_less_than_int(OBX_query_builder* builder, obx_schema_id property_id, int64_t value); +obx_qb_less_than_int = c_fn('obx_qb_less_than_int', obx_qb_cond, [OBX_query_builder_p, obx_schema_id, ctypes.c_int64]) + +# OBX_C_API obx_qb_cond obx_qb_less_or_equal_int(OBX_query_builder* builder, obx_schema_id property_id, int64_t value); +obx_qb_less_or_equal_int = c_fn('obx_qb_less_or_equal_int', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_int64]) + +# OBX_C_API obx_qb_cond obx_qb_between_2ints(OBX_query_builder* builder, obx_schema_id property_id, int64_t value_a, +# int64_t value_b); +obx_qb_between_2ints = c_fn('obx_qb_between_2ints', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_int64, ctypes.c_int64]) + +# OBX_C_API obx_qb_cond obx_qb_in_int64s(OBX_query_builder* builder, obx_schema_id property_id, const int64_t values[], +# size_t count); +obx_qb_in_int64s = c_fn('obx_qb_in_int64s', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.POINTER(ctypes.c_int64), ctypes.c_size_t]) + +# OBX_C_API obx_qb_cond obx_qb_not_in_int64s(OBX_query_builder* builder, obx_schema_id property_id, +# const int64_t values[], size_t count); +obx_qb_not_in_int64s = c_fn('obx_qb_not_in_int64s', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.POINTER(ctypes.c_int64), ctypes.c_size_t]) + +# OBX_C_API obx_qb_cond obx_qb_in_int32s(OBX_query_builder* builder, obx_schema_id property_id, const int32_t values[], +# size_t count); +obx_qb_in_int32s = c_fn('obx_qb_in_int32s', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.POINTER(ctypes.c_int32), ctypes.c_size_t]) + +# OBX_C_API obx_qb_cond obx_qb_not_in_int32s(OBX_query_builder* builder, obx_schema_id property_id, +# const int32_t values[], size_t count); +obx_qb_not_in_int32s = c_fn('obx_qb_not_in_int32s', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.POINTER(ctypes.c_int32), ctypes.c_size_t]) + +# OBX_C_API obx_qb_cond obx_qb_greater_than_double(OBX_query_builder* builder, obx_schema_id property_id, double value); +obx_qb_greater_than_double = c_fn('obx_qb_greater_than_double', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_double]) + +# OBX_C_API obx_qb_cond obx_qb_greater_or_equal_double(OBX_query_builder* builder, obx_schema_id property_id, +# double value); +obx_qb_greater_or_equal_double = c_fn('obx_qb_greater_or_equal_double', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_double]) + +# OBX_C_API obx_qb_cond obx_qb_less_than_double(OBX_query_builder* builder, obx_schema_id property_id, double value); +obx_qb_less_than_double = c_fn('obx_qb_less_than_double', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_double]) + +# OBX_C_API obx_qb_cond obx_qb_less_or_equal_double(OBX_query_builder* builder, obx_schema_id property_id, double value); +obx_qb_less_or_equal_double = c_fn('obx_qb_less_or_equal_double', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_double]) + +# OBX_C_API obx_qb_cond obx_qb_between_2doubles(OBX_query_builder* builder, obx_schema_id property_id, double value_a, +# double value_b); +obx_qb_between_2doubles = c_fn('obx_qb_between_2doubles', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_double, ctypes.c_double]) + +# OBX_C_API obx_qb_cond obx_qb_equals_bytes(OBX_query_builder* builder, obx_schema_id property_id, const void* value, +# size_t size); +obx_qb_equals_bytes = c_fn('obx_qb_equals_bytes', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_void_p, ctypes.c_size_t]) + +# OBX_C_API obx_qb_cond obx_qb_greater_than_bytes(OBX_query_builder* builder, obx_schema_id property_id, +# const void* value, size_t size); +obx_qb_greater_than_bytes = c_fn('obx_qb_greater_than_bytes', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_void_p, ctypes.c_size_t]) + +# OBX_C_API obx_qb_cond obx_qb_greater_or_equal_bytes(OBX_query_builder* builder, obx_schema_id property_id, +# const void* value, size_t size); +obx_qb_greater_or_equal_bytes = c_fn('obx_qb_greater_or_equal_bytes', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_void_p, ctypes.c_size_t]) + +# OBX_C_API obx_qb_cond obx_qb_less_than_bytes(OBX_query_builder* builder, obx_schema_id property_id, const void* value, +# size_t size); +obx_qb_less_than_bytes = c_fn('obx_qb_less_than_bytes', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_void_p, ctypes.c_size_t]) + +# OBX_C_API obx_qb_cond obx_qb_less_or_equal_bytes(OBX_query_builder* builder, obx_schema_id property_id, +# const void* value, size_t size); +obx_qb_less_or_equal_bytes = c_fn('obx_qb_less_or_equal_bytes', obx_qb_cond, + [OBX_query_builder_p, obx_schema_id, ctypes.c_void_p, ctypes.c_size_t]) + +# OBX_C_API obx_qb_cond obx_qb_all(OBX_query_builder* builder, const obx_qb_cond conditions[], size_t count); +obx_qb_all = c_fn('obx_qb_all', obx_qb_cond, [OBX_query_builder_p, obx_qb_cond_p, ctypes.c_size_t]) + +# OBX_C_API obx_qb_cond obx_qb_any(OBX_query_builder* builder, const obx_qb_cond conditions[], size_t count); +obx_qb_any = c_fn('obx_qb_any', obx_qb_cond, [OBX_query_builder_p, obx_qb_cond_p, ctypes.c_size_t]) + +# OBX_C_API obx_err obx_qb_param_alias(OBX_query_builder* builder, const char* alias); +obx_qb_param_alias = c_fn_rc('obx_qb_param_alias', [OBX_query_builder_p, ctypes.c_char_p]) + +# OBX_C_API obx_err obx_query_param_string(OBX_query* query, obx_schema_id entity_id, obx_schema_id property_id, const char* value); +obx_query_param_string = c_fn_rc('obx_query_param_string', [OBX_query_p, obx_schema_id, obx_schema_id, ctypes.c_char_p]) + +# OBX_C_API obx_err obx_query_param_int(OBX_query* query, obx_schema_id entity_id, obx_schema_id property_id, int64_t value); +obx_query_param_int = c_fn_rc('obx_query_param_int', [OBX_query_p, obx_schema_id, obx_schema_id, ctypes.c_int64]) + +# OBX_C_API obx_err obx_query_param_vector_float32(OBX_query* query, obx_schema_id entity_id, obx_schema_id property_id, const float* value, size_t element_count); +obx_query_param_vector_float32 = c_fn_rc('obx_query_param_vector_float32', + [OBX_query_p, obx_schema_id, obx_schema_id, ctypes.POINTER(ctypes.c_float), + ctypes.c_size_t]) + +# OBX_C_API obx_err obx_query_param_alias_vector_float32(OBX_query* query, const char* alias, const float* value, size_t element_count); +obx_query_param_alias_vector_float32 = c_fn_rc('obx_query_param_alias_vector_float32', + [OBX_query_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_float), + ctypes.c_size_t]) + +# OBX_C_API obx_err obx_query_param_alias_string(OBX_query* query, const char* alias, const char* value); +obx_query_param_alias_string = c_fn_rc('obx_query_param_alias_string', [OBX_query_p, ctypes.c_char_p, ctypes.c_char_p]) + +# OBX_C_API obx_err obx_query_param_alias_int(OBX_query* query, const char* alias, int64_t value); +obx_query_param_alias_int = c_fn_rc('obx_query_param_alias_int', [OBX_query_p, ctypes.c_char_p, ctypes.c_int64]) + +# OBX_C_API obx_err obx_qb_order(OBX_query_builder* builder, obx_schema_id property_id, OBXOrderFlags flags); +obx_qb_order = c_fn_rc('obx_qb_order', [OBX_query_builder_p, obx_schema_id, OBXOrderFlags]) + +# OBX_C_API obx_qb_cond obx_qb_nearest_neighbors_f32(OBX_query_builder* builder, obx_schema_id vector_property_id, const float* query_vector, size_t max_result_count) +obx_qb_nearest_neighbors_f32 = c_fn_qb_cond('obx_qb_nearest_neighbors_f32', + [OBX_query_builder_p, obx_schema_id, ctypes.POINTER(ctypes.c_float), + ctypes.c_size_t]) + +# OBX_C_API OBX_query* obx_query(OBX_query_builder* builder); +obx_query = c_fn('obx_query', OBX_query_p, [OBX_query_builder_p]) + +# OBX_C_API obx_err obx_query_close(OBX_query* query); +obx_query_close = c_fn_rc('obx_query_close', [OBX_query_p]) + +# OBX_C_API OBX_query* obx_query_clone(OBX_query* query); +obx_query_clone = c_fn('obx_query_clone', OBX_query_p, [OBX_query_p]) + +# OBX_C_API obx_err obx_query_offset(OBX_query* query, size_t offset); +obx_query_offset = c_fn_rc('obx_query_offset', [OBX_query_p, ctypes.c_size_t]) + +# OBX_C_API obx_err obx_query_offset_limit(OBX_query* query, size_t offset, size_t limit); +obx_query_offset_limit = c_fn_rc('obx_query_offset_limit', [OBX_query_p, ctypes.c_size_t, ctypes.c_size_t]) + +# OBX_C_API obx_err obx_query_limit(OBX_query* query, size_t limit); +obx_query_limit = c_fn_rc('obx_query_limit', [OBX_query_p, ctypes.c_size_t]) + +# OBX_C_API OBX_bytes_array* obx_query_find(OBX_query* query); +obx_query_find = c_fn('obx_query_find', OBX_bytes_array_p, [OBX_query_p]) + +# OBX_C_API obx_err obx_query_find_first(OBX_query* query, const void** data, size_t* size); +obx_query_find_first = c_fn_rc('obx_query_find_first', + [OBX_query_p, ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(ctypes.c_size_t)]) + +# OBX_C_API obx_err obx_query_find_unique(OBX_query* query, const void** data, size_t* size); +obx_query_find_unique = c_fn_rc('obx_query_find_unique', + [OBX_query_p, ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(ctypes.c_size_t)]) + +# OBX_C_API OBX_bytes_score_array* obx_query_find_with_scores(OBX_query* query); +obx_query_find_with_scores = c_fn('obx_query_find_with_scores', OBX_bytes_score_array_p, [OBX_query_p]) + +# typedef bool obx_data_visitor(void* user_data, const void* data, size_t size); + +# OBX_C_API obx_err obx_query_visit(OBX_query* query, obx_data_visitor* visitor, void* user_data); +# obx_query_visit = fn('obx_query_visit', obx_err, [OBX_query_p, obx_data_visitor_p, ctypes.c_void_p]) + +# OBX_C_API OBX_id_array* obx_query_find_ids(OBX_query* query); +obx_query_find_ids = c_fn('obx_query_find_ids', OBX_id_array_p, [OBX_query_p]) + +# OBX_C_API OBX_id_score_array* obx_query_find_ids_with_scores(OBX_query* query); +obx_query_find_ids_with_scores = c_fn('obx_query_find_ids_with_scores', OBX_id_score_array_p, [OBX_query_p]) + +# OBX_C_API OBX_id_array* obx_query_find_ids_by_score(OBX_query* query); +obx_query_find_ids_by_score = c_fn('obx_query_find_ids_by_score', OBX_id_array_p, [OBX_query_p]) + +# OBX_C_API obx_err obx_query_count(OBX_query* query, uint64_t* out_count); +obx_query_count = c_fn_rc('obx_query_count', [OBX_query_p, ctypes.POINTER(ctypes.c_uint64)]) + +# OBX_C_API obx_err obx_query_remove(OBX_query* query, uint64_t* out_count); +obx_query_remove = c_fn_rc('obx_query_remove', [OBX_query_p, ctypes.POINTER(ctypes.c_uint64)]) + +# OBX_C_API const char* obx_query_describe(OBX_query* query); +obx_query_describe = c_fn('obx_query_describe', ctypes.c_char_p, [OBX_query_p]) + +# OBX_C_API const char* obx_query_describe_params(OBX_query* query); +obx_query_describe_params = c_fn('obx_query_describe_params', ctypes.c_char_p, [OBX_query_p]) # OBX_bytes_array* (size_t count); -obx_bytes_array = fn('obx_bytes_array', OBX_bytes_array_p, [ctypes.c_size_t]) +obx_bytes_array = c_fn('obx_bytes_array', OBX_bytes_array_p, [ctypes.c_size_t]) # obx_err (OBX_bytes_array* array, size_t index, const void* data, size_t size); -obx_bytes_array_set = fn('obx_bytes_array_set', obx_err, - [OBX_bytes_array_p, ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t]) +obx_bytes_array_set = c_fn_rc('obx_bytes_array_set', [ + OBX_bytes_array_p, ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t]) # void (OBX_bytes_array * array); -obx_bytes_array_free = fn('obx_bytes_array_free', None, [OBX_bytes_array_p]) +obx_bytes_array_free = c_fn('obx_bytes_array_free', None, [OBX_bytes_array_p]) + +# OBX_C_API void obx_id_array_free(OBX_id_array* array); +obx_id_array_free = c_fn('obx_id_array_free', None, [OBX_id_array_p]) + +# OBX_C_API void obx_bytes_score_array_free(OBX_bytes_score_array* array) +obx_bytes_score_array_free = c_fn('obx_bytes_score_array_free', None, [OBX_bytes_score_array_p]) + +# OBX_C_API void obx_id_score_array_free(OBX_id_score_array* array) +obx_id_score_array_free = c_fn('obx_id_score_array_free', None, [OBX_id_score_array_p]) OBXPropertyType_Bool = 1 OBXPropertyType_Byte = 2 @@ -394,7 +926,16 @@ def c_voidp_as_bytes(voidp, size): OBXPropertyType_String = 9 OBXPropertyType_Date = 10 OBXPropertyType_Relation = 11 +OBXPropertyType_DateNano = 12 +OBXPropertyType_Flex = 13 +OBXPropertyType_BoolVector = 22 OBXPropertyType_ByteVector = 23 +OBXPropertyType_ShortVector = 24 +OBXPropertyType_CharVector = 25 +OBXPropertyType_IntVector = 26 +OBXPropertyType_LongVector = 27 +OBXPropertyType_FloatVector = 28 +OBXPropertyType_DoubleVector = 29 OBXPropertyType_StringVector = 30 OBXPropertyFlags_ID = 1 @@ -417,6 +958,11 @@ def c_voidp_as_bytes(voidp, size): OBXDebugFlags_LOG_QUERIES = 4 OBXDebugFlags_LOG_QUERY_PARAMETERS = 8 OBXDebugFlags_LOG_ASYNC_QUEUE = 16 +OBXDebugFlags_LOG_CACHE_HITS = 32 +OBXDebugFlags_LOG_CACHE_ALL = 64 +OBXDebugFlags_LOG_TREE = 128 +OBXDebugFlags_LOG_EXCEPTION_STACK_TRACE = 256 +OBXDebugFlags_RUN_THREADING_SELF_TEST = 512 # Standard put ("insert or update") OBXPutMode_PUT = 1 @@ -447,3 +993,27 @@ def c_voidp_as_bytes(voidp, size): # null values should be treated equal to zero (scalars only). OBXOrderFlags_NULLS_ZERO = 16 + +OBXHnswFlags_NONE = 0 +OBXHnswFlags_DEBUG_LOGS = 1 +OBXHnswFlags_DEBUG_LOGS_DETAILED = 2 +OBXHnswFlags_VECTOR_CACHE_SIMD_PADDING_OFF = 4 +OBXHnswFlags_REPARATION_LIMIT_CANDIDATES = 8 + +OBXVectorDistanceType_UNKNOWN = 0 +OBXVectorDistanceType_EUCLIDEAN = 1 +OBXVectorDistanceType_COSINE = 2 +OBXVectorDistanceType_DOT_PRODUCT = 3 +OBXVectorDistanceType_DOT_PRODUCT_NON_NORMALIZED = 10 + +OBXPutPaddingMode_PaddingAutomatic = 1 +OBXPutPaddingMode_PaddingAllowedByBuffer = 2 +OBXPutPaddingMode_PaddingByCaller = 3 + +OBXValidateOnOpenPagesFlags_None = 0 +OBXValidateOnOpenPagesFlags_VisitLeafPages = 1 + +OBXValidateOnOpenKvFlags_None = 0 + +OBXBackupRestoreFlags_None = 0 +OBXBackupRestoreFlags_OverwriteExistingData = 1 diff --git a/objectbox/condition.py b/objectbox/condition.py new file mode 100644 index 0000000..90d32f4 --- /dev/null +++ b/objectbox/condition.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +from enum import Enum +from typing import * +import numpy as np + +if TYPE_CHECKING: + from objectbox.c import obx_qb_cond + from objectbox.query_builder import QueryBuilder + + +class QueryCondition: + def and_(self, other: QueryCondition) -> QueryCondition: + return LogicQueryCondition(self, other, LogicQueryConditionOp.AND) + __and__ = and_ + + def or_(self, other: QueryCondition) -> QueryCondition: + return LogicQueryCondition(self, other, LogicQueryConditionOp.OR) + __or__ = or_ + + def apply(self, qb: QueryBuilder) -> obx_qb_cond: + """ Applies the QueryCondition to the supplied QueryBuilder. + + :return: + The C handle for the applied condition. + """ + raise NotImplementedError + + +class LogicQueryConditionOp(Enum): + AND = 1 + OR = 2 + + +class LogicQueryCondition(QueryCondition): + """ A QueryCondition describing a logical operation between two inner QueryCondition's (e.g. AND/OR). """ + + def __init__(self, cond1: QueryCondition, cond2: QueryCondition, op: LogicQueryConditionOp): + self._cond1 = cond1 + self._cond2 = cond2 + self._op = op + + def _apply_conditions(self, qb: QueryBuilder) -> List[obx_qb_cond]: + return [self._cond1.apply(qb), self._cond2.apply(qb)] + + def _apply_and(self, qb: QueryBuilder) -> obx_qb_cond: + return qb.all(self._apply_conditions(qb)) + + def _apply_or(self, qb: QueryBuilder) -> obx_qb_cond: + return qb.any(self._apply_conditions(qb)) + + def apply(self, qb: QueryBuilder) -> obx_qb_cond: + if self._op == LogicQueryConditionOp.AND: + return self._apply_and(qb) + elif self._op == LogicQueryConditionOp.OR: + return self._apply_or(qb) + else: + raise Exception(f"Unknown LogicQueryCondition op: {self._op.name}") + + +class PropertyQueryConditionOp(Enum): + EQ = 1 + NOT_EQ = 2 + CONTAINS = 3 + STARTS_WITH = 4 + ENDS_WITH = 5 + GT = 6 + GTE = 7 + LT = 8 + LTE = 9 + BETWEEN = 10 + NEAREST_NEIGHBOR = 11 + CONTAINS_KEY_VALUE = 12 + + +class PropertyQueryCondition(QueryCondition): + """ A QueryCondition describing an operation to be applied on a property (e.g. name == "John", age == 24) """ + + _OP_MAP: Dict[PropertyQueryConditionOp, str] = { + PropertyQueryConditionOp.EQ: "_apply_eq", + PropertyQueryConditionOp.NOT_EQ: "_apply_not_eq", + PropertyQueryConditionOp.CONTAINS: "_apply_contains", + PropertyQueryConditionOp.STARTS_WITH: "_apply_starts_with", + PropertyQueryConditionOp.ENDS_WITH: "_apply_ends_with", + PropertyQueryConditionOp.GT: "_apply_gt", + PropertyQueryConditionOp.GTE: "_apply_gte", + PropertyQueryConditionOp.LT: "_apply_lt", + PropertyQueryConditionOp.LTE: "_apply_lte", + PropertyQueryConditionOp.BETWEEN: "_apply_between", + PropertyQueryConditionOp.NEAREST_NEIGHBOR: "_apply_nearest_neighbor", + PropertyQueryConditionOp.CONTAINS_KEY_VALUE: "_contains_key_value" + # ... new property query conditions here ... :) + } + + def __init__(self, property_id: int, op: PropertyQueryConditionOp, args: Dict[str, Any]): + if op not in self._OP_MAP: + raise Exception(f"Invalid PropertyQueryConditionOp: {op}") + op_func_name = self._OP_MAP[op] + if not hasattr(self, op_func_name): + raise Exception(f"Missing PropertyQueryCondition op function: {op_func_name} (op: {op})") + op_func = getattr(self, op_func_name) + + self._property_id = property_id + self._op = op + self._op_func = op_func + self._args = args + self._alias = None + + def alias(self, value: str): + """ Sets an alias for this condition that can later be used with Query's set_parameter_* methods. """ + self._alias = value + return self + + def _apply_eq(self, qb: QueryBuilder) -> obx_qb_cond: + value = self._args['value'] + if isinstance(value, str): + case_sensitive = self._args['case_sensitive'] + return qb.equals_string(self._property_id, value, case_sensitive) + elif isinstance(value, int): + return qb.equals_int(self._property_id, value) + elif isinstance(value, bytes): + return qb.equals_bytes(self._property_id, value) + else: + raise Exception(f"Unsupported type for 'EQ': {type(value)}") + + def _apply_not_eq(self, qb: QueryBuilder) -> obx_qb_cond: + value = self._args['value'] + if isinstance(value, str): + case_sensitive = self._args['case_sensitive'] + return qb.not_equals_string(self._property_id, value, case_sensitive) + elif isinstance(value, int): + return qb.not_equals_int(self._property_id, value) + else: + raise Exception(f"Unsupported type for 'NOT_EQ': {type(value)}") + + def _apply_contains(self, qb: QueryBuilder) -> obx_qb_cond: + value = self._args['value'] + case_sensitive = self._args['case_sensitive'] + if isinstance(value, str): + return qb.contains_string(self._property_id, value, case_sensitive) + else: + raise Exception(f"Unsupported type for 'CONTAINS': {type(value)}") + + def _apply_starts_with(self, qb: QueryBuilder) -> obx_qb_cond: + value = self._args['value'] + case_sensitive = self._args['case_sensitive'] + if isinstance(value, str): + return qb.starts_with_string(self._property_id, value, case_sensitive) + else: + raise Exception(f"Unsupported type for 'STARTS_WITH': {type(value)}") + + def _apply_ends_with(self, qb: QueryBuilder) -> obx_qb_cond: + value = self._args['value'] + if isinstance(value, str): + case_sensitive = self._args['case_sensitive'] + return qb.ends_with_string(self._property_id, value, case_sensitive) + else: + raise Exception(f"Unsupported type for 'ENDS_WITH': {type(value)}") + + def _apply_gt(self, qb: QueryBuilder) -> obx_qb_cond: + value = self._args['value'] + if isinstance(value, str): + case_sensitive = self._args['case_sensitive'] + return qb.greater_than_string(self._property_id, value, case_sensitive) + elif isinstance(value, int): + return qb.greater_than_int(self._property_id, value) + elif isinstance(value, float): + return qb.greater_than_double(self._property_id, value) + elif isinstance(value, bytes): + return qb.greater_than_bytes(self._property_id, value) + else: + raise Exception(f"Unsupported type for 'GT': {type(value)}") + + def _apply_gte(self, qb: QueryBuilder) -> obx_qb_cond: + value = self._args['value'] + if isinstance(value, str): + case_sensitive = self._args['case_sensitive'] + return qb.greater_or_equal_string(self._property_id, value, case_sensitive) + elif isinstance(value, int): + return qb.greater_or_equal_int(self._property_id, value) + elif isinstance(value, float): + return qb.greater_or_equal_double(self._property_id, value) + elif isinstance(value, bytes): + return qb.greater_or_equal_bytes(self._property_id, value) + else: + raise Exception(f"Unsupported type for 'GTE': {type(value)}") + + def _apply_lt(self, qb: QueryBuilder) -> obx_qb_cond: + value = self._args['value'] + if isinstance(value, str): + case_sensitive = self._args['case_sensitive'] + return qb.less_than_string(self._property_id, value, case_sensitive) + elif isinstance(value, int): + return qb.less_than_int(self._property_id, value) + elif isinstance(value, float): + return qb.less_than_double(self._property_id, value) + elif isinstance(value, bytes): + return qb.less_than_bytes(self._property_id, value) + else: + raise Exception("Unsupported type for 'LT': " + str(type(value))) + + def _apply_lte(self, qb: QueryBuilder) -> obx_qb_cond: + value = self._args['value'] + if isinstance(value, str): + case_sensitive = self._args['case_sensitive'] + return qb.less_or_equal_string(self._property_id, value, case_sensitive) + elif isinstance(value, int): + return qb.less_or_equal_int(self._property_id, value) + elif isinstance(value, float): + return qb.less_or_equal_double(self._property_id, value) + elif isinstance(value, bytes): + return qb.less_or_equal_bytes(self._property_id, value) + else: + raise Exception(f"Unsupported type for 'LTE': {type(value)}") + + def _apply_between(self, qb: QueryBuilder) -> obx_qb_cond: + a = self._args['a'] + b = self._args['b'] + if isinstance(a, int) and isinstance(b, int): + return qb.between_2ints(self._property_id, a, b) + elif isinstance(a, float) or isinstance(b, float): + return qb.between_2doubles(self._property_id, a, b) + else: + raise Exception(f"Unsupported type for 'BETWEEN': {type(a)}") + + def _apply_nearest_neighbor(self, qb: QueryBuilder) -> obx_qb_cond: + query_vector = self._args['query_vector'] + element_count = self._args['element_count'] + + if len(query_vector) == 0: + raise Exception("query_vector can't be empty") + + is_float_vector = False + is_float_vector |= isinstance(query_vector, np.ndarray) and query_vector.dtype == np.float32 + is_float_vector |= isinstance(query_vector, list) and type(query_vector[0]) == float + if is_float_vector: + return qb.nearest_neighbors_f32(self._property_id, query_vector, element_count) + else: + raise Exception(f"Unsupported type for 'NEAREST_NEIGHBOR': {type(query_vector)}") + + def _contains_key_value(self, qb: QueryBuilder) -> obx_qb_cond: + key = self._args['key'] + value = self._args['value'] + case_sensitive = self._args['case_sensitive'] + return qb.contains_key_value(self._property_id, key, value, case_sensitive) + + def apply(self, qb: QueryBuilder) -> obx_qb_cond: + c_cond = self._op_func(qb) + if self._alias is not None: + qb.alias(self._alias) + return c_cond diff --git a/objectbox/logger.py b/objectbox/logger.py new file mode 100644 index 0000000..2ce53d9 --- /dev/null +++ b/objectbox/logger.py @@ -0,0 +1,18 @@ +import sys +import logging + +logger = logging.getLogger("objectbox") + + +def setup_stdout_logger(): + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(logging.DEBUG) + # Output format example: + # 2024-04-04 10:16:46,272 [objectbox-py] [DEBUG] Creating property "id" (ID=1, UID=1001) + formatter = logging.Formatter('%(asctime)s [objectbox-py] [%(levelname)-5s] %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + return logger + +# Not need to hook stdout as pytest will do the job. Use --log-cli-level= to set log level +# setup_stdout_logger() diff --git a/objectbox/model/__init__.py b/objectbox/model/__init__.py index 69a2e19..113eeb5 100644 --- a/objectbox/model/__init__.py +++ b/objectbox/model/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 ObjectBox Ltd. All rights reserved. +# Copyright 2019-2024 ObjectBox Ltd. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,8 +20,40 @@ __all__ = [ 'Model', 'Entity', - 'Id', 'IdUid', 'Property', - 'PropertyType' + 'PropertyType', + 'Id', + 'IdUid', + 'Bool', + 'String', + 'Int8', + 'Int16', + 'Int32', + 'Int64', + 'Float32', + 'Float64', + 'Date', + 'DateNano', + 'Flex', + 'BoolVector', + 'Int8Vector', + 'Int16Vector', + 'CharVector', + 'Int32Vector', + 'Int64Vector', + 'Float32Vector', + 'Float64Vector', + 'Index', + 'HnswIndex', + 'VectorDistanceType', + 'BoolList', + 'Int8List', + 'Int16List', + 'CharList', + 'Int32List', + 'Int64List', + 'Float32List', + 'Float64List', + 'Bytes', ] diff --git a/objectbox/model/entity.py b/objectbox/model/entity.py index 2f83e59..6c486db 100644 --- a/objectbox/model/entity.py +++ b/objectbox/model/entity.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 ObjectBox Ltd. All rights reserved. +# Copyright 2019-2024 ObjectBox Ltd. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,109 +11,199 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import inspect import flatbuffers +import flatbuffers.flexbuffers +import numpy as np +from datetime import datetime, timezone +import logging from objectbox.c import * +from objectbox.model.iduid import IdUid from objectbox.model.properties import Property +from objectbox.utils import date_value_to_int +import threading # _Entity class holds model information as well as conversions between python objects and FlatBuffers (ObjectBox data) class _Entity(object): - def __init__(self, cls, id: int, uid: int): - # currently, ID and UID are mandatory and are not fetched from the model.json - if id <= 0: - raise Exception( - "invalid or no 'id; given in the @Entity annotation") - - if uid <= 0: - raise Exception( - "invalid or no 'uid' given in the @Entity annotation") - - self.cls = cls - self.name = cls.__name__ - self.id = id - self.uid = uid - - self.last_property_id = None # IdUid - set in model.entity() - - self.properties = list() # List[Property] - self.offset_properties = list() # List[Property] - self.id_property = None - self.fill_properties() - - def __call__(self, *args): - return self.cls(*args) - - def fill_properties(self): + def __init__(self, user_type, uid: int = 0): + self._user_type = user_type + self._iduid = IdUid(0, uid) + self._name = user_type.__name__ + self._last_property_iduid = IdUid(0, 0) + + self._properties: List[Property] = list() # List[Property] + self._offset_properties = list() # List[Property] + self._id_property = None + self._fill_properties() + self._tl = threading.local() + + @property + def _id(self) -> int: + return self._iduid.id + + @property + def _uid(self) -> int: + return self._iduid.uid + + def _has_uid(self) -> bool: + return self._iduid.uid != 0 + + def _on_sync(self): + """ Method called once ID/UID are synced with the model file. """ + assert self._iduid.is_assigned() + for prop in self._properties: + prop.on_sync() + + def __call__(self, **properties): + """ The constructor of the user Entity class. """ + object_ = self._user_type() + for prop_name, prop_val in properties.items(): + if not hasattr(object_, prop_name): + raise Exception(f"Entity {self._name} has no property \"{prop_name}\"") + setattr(object_, prop_name, prop_val) + return object_ + + def __getattr__(self, name): + """ Overload to get properties via "." notation. """ + for prop in self._properties: + if prop.name == name: + return prop + return self.__getattribute__(name) + + def _fill_properties(self): # TODO allow subclassing and support entities with __slots__ defined - variables = dict(vars(self.cls)) + variables = dict(vars(self._user_type)) # filter only subclasses of Property variables = {k: v for k, v in variables.items( ) if issubclass(type(v), Property)} - for k, prop in variables.items(): - prop._name = k - self.properties.append(prop) + for prop_name, prop in variables.items(): + prop.name = prop_name + self._properties.append(prop) - if prop._is_id: - if self.id_property: - raise Exception("duplicate ID property: '%s' and '%s'" % ( - self.id_property._name, prop._name)) - self.id_property = prop + if prop.is_id(): + if self._id_property: + raise Exception(f"Duplicate ID property: \"{self._id_property.name}\" and \"{prop.name}\"") + self._id_property = prop if prop._fb_type == flatbuffers.number_types.UOffsetTFlags: - assert prop._ob_type in [OBXPropertyType_String, OBXPropertyType_ByteVector], \ - "programming error - invalid type OB & FB type combination" - self.offset_properties.append(prop) - - # print('Property {}.{}: {} (ob:{} fb:{})'.format(self.name, prop._name, prop._py_type, prop._ob_type, prop._fb_type)) - - if not self.id_property: + assert prop._ob_type in [ + OBXPropertyType_String, + OBXPropertyType_BoolVector, + OBXPropertyType_ByteVector, + OBXPropertyType_ShortVector, + OBXPropertyType_CharVector, + OBXPropertyType_IntVector, + OBXPropertyType_LongVector, + OBXPropertyType_FloatVector, + OBXPropertyType_DoubleVector, + OBXPropertyType_Flex, + ], "programming error - invalid type OB & FB type combination" + self._offset_properties.append(prop) + + # print('Property {}.{}: {} (ob:{} fb:{})'.format(self._name, prop.name, prop._py_type, prop._ob_type, prop._fb_type)) + + if not self._id_property: raise Exception("ID property is not defined") - elif self.id_property._ob_type != OBXPropertyType_Long: + elif self._id_property._ob_type != OBXPropertyType_Long: raise Exception("ID property must be an int") - def get_value(self, object, prop: Property): + def _get_property(self, name: str): + """ Gets the property having the given name. """ + for prop in self._properties: + if prop.name == name: + return prop + raise Exception(f"Property \"{name}\" not found in Entity: \"{self._name}\"") + + def _get_property_id(self, prop: Union[int, str, Property]) -> int: + """ A convenient way to get the property ID regardless having its ID, name or Property. """ + if isinstance(prop, int): + return prop # We already have it! + elif isinstance(prop, str): + return self._get_property(prop).id + elif isinstance(prop, Property): + return prop.id + else: + raise Exception(f"Unsupported Property type: {type(prop)}") + + def _get_value(self, object, prop: Property): # in case value is not overwritten on the object, it's the Property object itself (= as defined in the Class) - val = getattr(object, prop._name) - if val == prop: - return prop._py_type() # default (empty) value for the given type + val = getattr(object, prop.name) + if prop._py_type == np.ndarray: + if (val == np.array(prop)).all(): + return np.array([]) + elif val == prop: + if prop._ob_type == OBXPropertyType_Date or prop._ob_type == OBXPropertyType_DateNano: + return 0.0 # For marshalling, prefer float over datetime + elif prop._ob_type == OBXPropertyType_Flex: + return None + else: + return prop._py_type() # default (empty) value for the given type return val - def get_object_id(self, object) -> int: - return self.get_value(object, self.id_property) + def _get_object_id(self, obj) -> int: + return self._get_value(obj, self._id_property) - def set_object_id(self, object, id: int): - setattr(object, self.id_property._name, id) + def _set_object_id(self, obj, id_: int): + setattr(obj, self._id_property.name, id_) - def marshal(self, object, id: int) -> bytearray: - builder = flatbuffers.Builder(256) + def _marshal(self, object, id: int) -> bytearray: + if not hasattr(self._tl, "builder"): + self._tl.builder = flatbuffers.Builder(256) + builder = self._tl.builder + builder.Clear() # prepare some properties that need to be built in FB before starting the main object offsets = {} - for prop in self.offset_properties: - val = self.get_value(object, prop) + for prop in self._offset_properties: + val = self._get_value(object, prop) if prop._ob_type == OBXPropertyType_String: - offsets[prop._id] = builder.CreateString(val.encode('utf-8')) + offsets[prop.id] = builder.CreateString(val.encode('utf-8')) + elif prop._ob_type == OBXPropertyType_BoolVector: + # Using a numpy bool as it seems to be more consistent in terms of size. TBD + # https://numpy.org/doc/stable/reference/arrays.scalars.html#numpy.bool + offsets[prop.id] = builder.CreateNumpyVector(np.array(val, dtype=np.bool_)) elif prop._ob_type == OBXPropertyType_ByteVector: - offsets[prop._id] = builder.CreateByteVector(val) + offsets[prop.id] = builder.CreateByteVector(val) + elif prop._ob_type == OBXPropertyType_ShortVector: + offsets[prop.id] = builder.CreateNumpyVector(np.array(val, dtype=np.int16)) + elif prop._ob_type == OBXPropertyType_CharVector: + offsets[prop.id] = builder.CreateNumpyVector(np.array(val, dtype=np.uint16)) + elif prop._ob_type == OBXPropertyType_IntVector: + offsets[prop.id] = builder.CreateNumpyVector(np.array(val, dtype=np.int32)) + elif prop._ob_type == OBXPropertyType_LongVector: + offsets[prop.id] = builder.CreateNumpyVector(np.array(val, dtype=np.int64)) + elif prop._ob_type == OBXPropertyType_FloatVector: + offsets[prop.id] = builder.CreateNumpyVector(np.array(val, dtype=np.float32)) + elif prop._ob_type == OBXPropertyType_DoubleVector: + offsets[prop.id] = builder.CreateNumpyVector(np.array(val, dtype=np.float64)) + elif prop._ob_type == OBXPropertyType_Flex: + flex_builder = flatbuffers.flexbuffers.Builder() + flex_builder.Add(val) + buffer = flex_builder.Finish() + offsets[prop.id] = builder.CreateByteVector(bytes(buffer)) else: assert False, "programming error - invalid type OB & FB type combination" # start the FlatBuffers object with the largest number of properties that were ever present in the Entity - builder.StartObject(self.last_property_id.id) + builder.StartObject(self._last_property_iduid.id) # add properties to the FB object - for prop in self.properties: - if prop._id in offsets: - val = offsets[prop._id] + for prop in self._properties: + prop_id = prop.id + if prop_id in offsets: + val = offsets[prop_id] if val: builder.PrependUOffsetTRelative(val) else: - val = id if prop == self.id_property else self.get_value( - object, prop) + val = id if prop == self._id_property else self._get_value(object, prop) + if prop._ob_type == OBXPropertyType_Date: + val = date_value_to_int(val, 1000) # convert to milliseconds + elif prop._ob_type == OBXPropertyType_DateNano: + val = date_value_to_int(val, 1000000000) # convert to nanoseconds builder.Prepend(prop._fb_type, val) builder.Slot(prop._fb_slot) @@ -121,41 +211,100 @@ def marshal(self, object, id: int) -> bytearray: builder.Finish(builder.EndObject()) return builder.Output() - def unmarshal(self, data: bytes): + def _unmarshal(self, data: bytes): pos = flatbuffers.encode.Get(flatbuffers.packer.uoffset, data, 0) table = flatbuffers.Table(data, pos) # initialize an empty object - obj = self.cls() + obj = self._user_type() # fill it with the data read from FlatBuffers - for prop in self.properties: + for prop in self._properties: o = table.Offset(prop._fb_v_offset) val = None + ob_type = prop._ob_type if not o: val = prop._py_type() # use default (empty) value if not present in the object - elif prop._ob_type == OBXPropertyType_String: + elif ob_type == OBXPropertyType_String: val = table.String(o + table.Pos).decode('utf-8') - elif prop._ob_type == OBXPropertyType_ByteVector: + elif ob_type == OBXPropertyType_BoolVector: + val = table.GetVectorAsNumpy(flatbuffers.number_types.BoolFlags, o) + elif ob_type == OBXPropertyType_ByteVector: # access the FB byte vector information start = table.Vector(o) size = table.VectorLen(o) - # slice the vector as a requested type - val = prop._py_type(table.Bytes[start:start+size]) + val = prop._py_type(table.Bytes[start:start + size]) + elif ob_type == OBXPropertyType_ShortVector: + val = table.GetVectorAsNumpy(flatbuffers.number_types.Int16Flags, o) + elif ob_type == OBXPropertyType_CharVector: + val = table.GetVectorAsNumpy(flatbuffers.number_types.Int16Flags, o) + elif ob_type == OBXPropertyType_Date: + val = table.Get(prop._fb_type, o + table.Pos) # int + if prop._py_type == datetime: + val = datetime.fromtimestamp(val / 1000.0, tz=timezone.utc) + elif prop._py_type == float: + val = val / 1000.0 + elif ob_type == OBXPropertyType_DateNano and prop._py_type == datetime: + val = table.Get(prop._fb_type, o + table.Pos) # int + if prop._py_type == datetime: + val = datetime.fromtimestamp(val / 1000000000.0, tz=timezone.utc) + elif prop._py_type == float: + val = val / 1000000000.0 + elif ob_type == OBXPropertyType_IntVector: + val = table.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o) + elif ob_type == OBXPropertyType_LongVector: + val = table.GetVectorAsNumpy(flatbuffers.number_types.Int64Flags, o) + elif ob_type == OBXPropertyType_FloatVector: + val = table.GetVectorAsNumpy(flatbuffers.number_types.Float32Flags, o) + elif ob_type == OBXPropertyType_DoubleVector: + val = table.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o) + elif ob_type == OBXPropertyType_Flex: + # access the FB byte vector information + start = table.Vector(o) + size = table.VectorLen(o) + # slice the vector as bytes + buf = table.Bytes[start:start + size] + val = flatbuffers.flexbuffers.Loads(buf) else: val = table.Get(prop._fb_type, o + table.Pos) - - setattr(obj, prop._name, val) + if prop._py_type == list: + val = val.tolist() + setattr(obj, prop.name, val) return obj - -# entity decorator - wrap _Entity to allow @Entity(id=, uid=), i.e. no class argument -def Entity(cls=None, id: int = 0, uid: int = 0): - if cls: - return _Entity(cls, id, uid) - else: - def wrapper(cls): - return _Entity(cls, id, uid) - - return wrapper +# Dictionary of entity types (metadata) collected by the Entity decorator. +# Note: using a list not a set to keep the order of entities as they were defined (set would not be deterministic). +obx_models_by_name: Dict[str, List[_Entity]] = {} + + +def Entity(uid: int = 0, model: str = "default") -> Callable[[Type], _Entity]: + """ Entity decorator that wraps _Entity to allow @Entity(id=, uid=); i.e. no class arguments. """ + + def wrapper(class_): + # Also allow defining properties as class members; we'll instantiate them here + class_members = inspect.getmembers(class_, lambda a: (inspect.isclass(a) and issubclass(a, Property))) + for name, member_type in class_members: + assert issubclass(member_type, Property) + # noinspection PyArgumentList + obj = member_type() # Subclasses of Property have no constructor arguments + setattr(class_, name, obj) + + types = obx_models_by_name.get(model) + if types is None: + types = [] + obx_models_by_name[model] = types + + entity_type = _Entity(class_, uid) + for existing in types: + if existing._name == entity_type._name: + # OK for tests, where multiple models are created with the same entity name + logging.warning(f"Model \"{model}\" already contains an entity type \"{entity_type._name}\"; replacing it.") + types.remove(existing) + break + + obx_models_by_name[model].append(entity_type) + logging.info(f"Entity type {entity_type._name} added to model {model}") + return entity_type + + return wrapper diff --git a/objectbox/model/idsync.py b/objectbox/model/idsync.py new file mode 100644 index 0000000..f286191 --- /dev/null +++ b/objectbox/model/idsync.py @@ -0,0 +1,311 @@ +import random +from typing import * +from objectbox.logger import logger +from objectbox.model import Model +from objectbox.model.entity import _Entity +from objectbox.model.properties import Property, Index, HnswIndex +from objectbox.model.iduid import IdUid + +MODEL_PARSER_VERSION = 5 + + +class IdSync: + """ + Synchronizes a model with the IDs from model JSON file. + After syncing, the model will have all IDs assigned. + The JSON file is written (from scratch) based on the model. + """ + + def __init__(self, model: Model, model_json_filepath: str): + self.model = model + if len(model.entities) == 0: + raise ValueError("A valid model must have at least one entity") + + self.model_filepath = model_json_filepath + self.model_json = None + + self._assigned_uids: Set[int] = set() + + self._load_model_json() + + def _load_model_json(self): + import json + from os import path + + if not path.exists(self.model_filepath): + logger.debug(f"Model file not found: {self.model_filepath}") + return + + with open(self.model_filepath, "rt") as model_file: + self.model_json = json.load(model_file) + logger.debug(f"Syncing model with model file: {self.model_filepath}") + + self._load_assigned_uids() + + def _load_assigned_uids(self): + for entity_json in self.model_json["entities"]: + entity_uid = IdUid.from_str(entity_json["id"]).uid + if entity_uid in self._assigned_uids: + raise ValueError(f"An entity's UID {entity_uid} has already been used elsewhere") + self._assigned_uids.add(entity_uid) + + for prop_json in entity_json["properties"]: + prop_uid = IdUid.from_str(prop_json["id"]).uid + if prop_uid in self._assigned_uids: + raise ValueError(f"A property's UID {prop_uid} has already been used elsewhere") + self._assigned_uids.add(prop_uid) + + if "indexId" in prop_json: + index_uid = IdUid.from_str(prop_json["indexId"]).uid + if index_uid in self._assigned_uids: + raise ValueError(f"An index's UID {index_uid} has already been used elsewhere") + self._assigned_uids.add(index_uid) + + def _save_model_json(self): + """ Replaces model JSON with the serialized model whose ID/UIDs are assigned. """ + + # model.validate_ids_assigned() + + model_json = { + "_note1": "KEEP THIS FILE! Check it into a version control system (VCS) like git.", + "_note2": "ObjectBox manages crucial IDs for your object model. See docs for details.", + "_note3": "If you have VCS merge conflicts, you must resolve them according to ObjectBox docs.", + "modelVersionParserMinimum": MODEL_PARSER_VERSION, + "entities": [], + "lastEntityId": str(self.model.last_entity_iduid), + "lastIndexId": str(self.model.last_index_iduid) + } + # TODO lastRelationId + # TODO modelVersion + # TODO retiredEntityUids + # TODO retiredIndexUids + # TODO retiredPropertyUids + # TODO retiredRelationUids + # TODO version + + for entity in self.model.entities: + entity_json = { + "id": str(entity._iduid), + "name": entity._name, + "lastPropertyId": str(entity._last_property_iduid), + "properties": [] + } + for prop in entity._properties: + prop_json = { + "id": str(prop.iduid), + "name": prop.name, + "type": prop._ob_type, + } + if prop._flags != 0: + prop_json["flags"] = prop._flags + if prop.index is not None: + prop_json["indexId"] = str(prop.index.iduid) + entity_json["properties"].append(prop_json) + model_json["entities"].append(entity_json) + + import json + with open(self.model_filepath, "w") as model_file: + model_file.write(json.dumps(model_json, indent=2)) # Pretty + + # *** Sync *** + + def _find_entity_json_by_uid(self, uid: int) -> Optional[Dict[str, Any]]: + """ Finds entity JSON by UID. """ + if self.model_json is None: + return None + # TODO put entities in a dict (e.g. while/after loading) for faster lookup + for entity_json in self.model_json["entities"]: + if IdUid.from_str(entity_json["id"]).uid == uid: + return entity_json + return None + + def _find_entity_json_by_name(self, entity_name: str) -> Optional[Dict[str, Any]]: + """ Finds entity JSON by name. """ + if self.model_json is None: + return None + # TODO put entities in a dict (e.g. while/after loading) for faster lookup + for entity_json in self.model_json["entities"]: + if entity_json["name"] == entity_name: + return entity_json + return None + + def _find_property_json_by_uid(self, entity_json: Dict[str, Any], uid: int) -> Optional[Dict[str, Any]]: + """ Finds entity property JSON by property UID. """ + # TODO put properties in a multi-dict (e.g. while/after loading) for faster lookup + for prop_json in entity_json["properties"]: + if IdUid.from_str(prop_json["id"]).uid == uid: + return prop_json + return None + + def _find_property_json_by_name(self, entity_json: Dict[str, Any], prop_name: str) -> Optional[Dict[str, Any]]: + """ Finds entity property JSON by property name. """ + # TODO put properties in a multi-dict (e.g. while/after loading) for faster lookup + for prop_json in entity_json["properties"]: + if prop_json["name"] == prop_name: + return prop_json + return None + + def _generate_uid(self) -> int: + while True: + generated_uid = random.getrandbits(63) + 1 # 0 would be invalid + if generated_uid not in self._assigned_uids: + break + self._assigned_uids.add(generated_uid) + return generated_uid + + def _validate_uid_unassigned(self, uid: int): + """ Validates that a user supplied UID is not assigned for any other entity/property/index. + Raises a ValueError if the UID is already assigned elsewhere. + """ + if uid in self._assigned_uids: + raise ValueError(f"User supplied UID {uid} is already assigned elsewhere") + + def _validate_matching_prop(self, entity: _Entity, prop: Property, prop_json: Dict[str, Any]): + """ Validates that the given property matches the JSON property. """ + try: + # Don't check name equality as the property could be matched by UID (rename) + # if validate_name and prop.name != prop_json["name"]: + # raise ValueError(f"name {prop.name} != name {prop_json['name']} (in JSON)") + if prop._ob_type != prop_json["type"]: + raise ValueError(f"OBX type {prop._ob_type} != OBX type {prop_json['type']} (in JSON)") + + json_flags = prop_json.get("flags", 0) + if prop._flags != json_flags: + raise ValueError(f"flags {prop._flags} != flags {json_flags} (in JSON)") + + if prop.index is None and "indexId" in prop_json: + raise ValueError("property hasn't index, but index found in JSON") + elif prop.index is not None and "indexId" not in prop_json: + raise ValueError("property has index, but index not found in JSON") + except ValueError as error: + raise ValueError(f"Property {entity._name}.{prop.name} mismatches property found in JSON file: {error}") + + def _sync_index(self, entity: _Entity, prop: Property, prop_json: Optional[Dict[str, Any]]) -> bool: + assert prop.index is not None + index = prop.index + + write_json = False + + # Fetch index ID/UID from JSON file + iduid_json = None + if (prop_json is not None) and ("indexId" in prop_json): + iduid_json = IdUid.from_str(prop_json["indexId"]) + + # User provided a UID not matching index's, make sure it's not assigned elsewhere + if index.has_uid() and (iduid_json is not None) and (index.uid != iduid_json.uid): + self._validate_uid_unassigned(index.uid) + + # Generate UID only if not supplied by the user, and index isn't found in JSON + if not index.has_uid() and iduid_json is None: + index.iduid.uid = self._generate_uid() + + if (iduid_json is not None) and (not index.has_uid() or index.iduid.uid == iduid_json.uid): # Load + index.iduid = IdUid.from_str(prop_json["indexId"]) + else: # Assign new ID to new index + index.iduid = IdUid(self.model.last_index_iduid.id + 1, index.uid) + self.model.last_index_iduid = index.iduid + write_json = True + + return write_json + + def _sync_property(self, entity: _Entity, prop: Property, entity_json: Optional[Dict[str, Any]]) -> bool: + write_json = False + + prop_json = None + if prop.has_uid(): + if entity_json is not None: + prop_json = self._find_property_json_by_uid(entity_json, prop.uid) + if prop_json is None: + # User provided a UID not matching any property (within the entity), make sure it's not assigned + # elsewhere + self._validate_uid_unassigned(prop.uid) + else: + write_json = prop.name != prop_json["name"] # If renaming we shall update the JSON + else: + if entity_json is not None: + prop_json = self._find_property_json_by_name(entity_json, prop.name) + + if prop_json is not None: # Load existing IDs from JSON + # Property was matched with a JSON property (either by UID or by name), make sure they're equal + self._validate_matching_prop(entity, prop, prop_json) + prop.iduid = IdUid.from_str(prop_json["id"]) + else: # Assign new ID to new property + if not prop.has_uid(): + prop.iduid.uid = self._generate_uid() + prop.iduid = IdUid(entity._last_property_iduid.id + 1, prop.iduid.uid) + entity._last_property_iduid = prop.iduid + write_json = True + + if prop.index is not None: + write_json |= self._sync_index(entity, prop, prop_json) + + return write_json + + def _sync_entity(self, entity: _Entity) -> bool: + write_json = False + + # entity_json = None + if entity._has_uid(): + entity_json = self._find_entity_json_by_uid(entity._uid) + if entity_json is None: + # User provided a UID not matching any entity, make sure it's not assigned elsewhere + self._validate_uid_unassigned(entity._uid) + else: + write_json = entity._name != entity_json["name"] # If renaming we shall update the JSON + else: + entity_json = self._find_entity_json_by_name(entity._name) + + # Write JSON if the number of properties differs (to handle removed property) + if entity_json is not None: + write_json |= len(entity._properties) != len(entity_json["properties"]) + + if entity_json is not None: # Load existing IDs from JSON + entity._iduid = IdUid.from_str(entity_json["id"]) + entity._last_property_iduid = IdUid.from_str(entity_json["lastPropertyId"]) + else: # Assign new ID to new entity + if not entity._has_uid(): + entity._iduid.uid = self._generate_uid() + entity._iduid = IdUid(self.model.last_entity_iduid.id + 1, entity._iduid.uid) + self.model.last_entity_iduid = entity._iduid + entity._last_property_iduid = IdUid(0, 0) + write_json = True + + # Load properties + for prop in entity._properties: + write_json |= self._sync_property(entity, prop, entity_json) + + return write_json + + def sync(self) -> bool: + """ Syncs the provided model with the model JSON file. + Returns True if the model JSON was written. """ + + if self.model_json is not None: + self.model.last_entity_iduid = IdUid.from_str(self.model_json["lastEntityId"]) + self.model.last_index_iduid = IdUid.from_str(self.model_json["lastIndexId"]) + # self.model.last_relation_iduid = + + write_json = False + + # Write JSON if the number of entities differs (to handle removed entity) + if self.model_json is not None: + write_json |= len(self.model_json["entities"]) != len(self.model.entities) + + for entity in self.model.entities: + write_json |= self._sync_entity(entity) + + if write_json: + logger.info(f"Model changed, writing model.json: {self.model_filepath}") + self._save_model_json() + + self.model.on_sync() # Notify model synced + + return write_json + + +def sync_model(model: Model, model_filepath: str = "objectbox-model.json") -> bool: + """ Syncs the provided model with the model JSON file. + Returns True if changes were made and the model JSON was written. """ + + id_sync = IdSync(model, model_filepath) + return id_sync.sync() diff --git a/objectbox/model/iduid.py b/objectbox/model/iduid.py new file mode 100644 index 0000000..09cb203 --- /dev/null +++ b/objectbox/model/iduid.py @@ -0,0 +1,29 @@ +class IdUid: + __slots__ = 'id', 'uid' + + def __init__(self, id_: int, uid: int): + self.id = id_ + self.uid = uid + + def is_assigned(self): + """ Checks that both ID and UID are assigned. Shall be true after the model is synced. """ + return self.id != 0 and self.uid != 0 + + def __bool__(self): + return self.is_assigned() + + def __eq__(self, rhs: 'IdUid'): + return self.id == rhs.id and self.uid == rhs.uid + + def __str__(self): + return f"{self.id}:{self.uid}" + + @staticmethod + def from_str(str_: str): + """ Parses IdUid from a string formatted like: "id:uid" """ + tmp = str_.split(":") + return IdUid(int(tmp[0]), int(tmp[1])) + + @staticmethod + def unassigned(): + return IdUid(0, 0) diff --git a/objectbox/model/model.py b/objectbox/model/model.py index 0d27950..3845bf2 100644 --- a/objectbox/model/model.py +++ b/objectbox/model/model.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 ObjectBox Ltd. All rights reserved. +# Copyright 2019-2024 ObjectBox Ltd. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,58 +13,104 @@ # limitations under the License. -from objectbox.model.entity import _Entity +from objectbox.logger import logger from objectbox.c import * - - -class IdUid: - __slots__ = 'id', 'uid' - - def __init__(self, id: int, uid: int): - self.id = id - self.uid = uid - - def __bool__(self): - return self.id != 0 or self.uid != 0 +from objectbox.model.iduid import IdUid +from objectbox.model.entity import _Entity +from objectbox.model.properties import * class Model: def __init__(self): - self._entities = list() - self._c_model = obx_model() - self.last_entity_id = IdUid(0, 0) - self.last_index_id = IdUid(0, 0) - self.last_relation_id = IdUid(0, 0) + self.entities: List[_Entity] = [] - def entity(self, entity: _Entity, last_property_id: IdUid): - if not isinstance(entity, _Entity): - raise Exception("Given type is not an Entity. Are you passing an instance instead of a type or did you " - "forget the '@Entity' annotation?") - - entity.last_property_id = last_property_id - - obx_model_entity(self._c_model, c_str( - entity.name), entity.id, entity.uid) - - for v in entity.properties: - obx_model_property(self._c_model, c_str( - v._name), v._ob_type, v._id, v._uid) - if v._flags != 0: - obx_model_property_flags(self._c_model, v._flags) + self.last_entity_iduid = IdUid(0, 0) + self.last_index_iduid = IdUid(0, 0) + self.last_relation_iduid = IdUid(0, 0) - obx_model_entity_last_property_id( - self._c_model, last_property_id.id, last_property_id.uid) + self._c_model = None - # called by Builder - def _finish(self): - if self.last_relation_id: - obx_model_last_relation_id( - self._c_model, self.last_relation_id.id, self.last_relation_id.uid) + def on_sync(self): + """ Method called once ID/UID are synced with the model file. """ + for entity in self.entities: + entity._on_sync() - if self.last_index_id: - obx_model_last_index_id( - self._c_model, self.last_index_id.id, self.last_index_id.uid) - - if self.last_entity_id: - obx_model_last_entity_id( - self._c_model, self.last_entity_id.id, self.last_entity_id.uid) + def entity(self, entity: _Entity): + if not isinstance(entity, _Entity): + raise Exception(f"The given type is not an Entity: {type(entity)}. " + f"Ensure to have an @Entity annotation on the class.") + for other_entity in self.entities: # Linear search (we should't have many entities) + if entity._name == other_entity._name: + raise Exception(f"Duplicate entity: \"{entity._name}\"") + self.entities.append(entity) + + def validate_ids_assigned(self): + # TODO validate last_relation_iduid + has_entities = len(self.entities) > 0 + has_indices = False + for entity in self.entities: + has_properties = len(entity._properties) > 0 + if not entity._iduid.is_assigned(): + raise ValueError(f"Entity \"{entity._name}\" ID/UID not assigned") + for prop in entity._properties: + if not prop.iduid.is_assigned(): + raise ValueError(f"Property \"{entity._name}.{prop.name}\" ID/UID not assigned") + if prop.index is not None: + has_indices = True + if not prop.index.iduid.is_assigned(): + raise ValueError(f"Property index \"{entity._name}.{prop.name}\" ID/UID not assigned") + if has_properties and not entity._last_property_iduid.is_assigned(): + raise ValueError(f"Entity \"{entity._name}\" last property ID/UID not assigned") + if has_entities and not self.last_entity_iduid.is_assigned(): + raise ValueError("Last entity ID/UID not assigned") + if has_indices and not self.last_index_iduid.is_assigned(): + raise ValueError("Last index ID/UID not assigned") + + def _set_hnsw_params(self, index: HnswIndex): + if index.dimensions is not None: + obx_model_property_index_hnsw_dimensions(self._c_model, index.dimensions) + if index.neighbors_per_node is not None: + obx_model_property_index_hnsw_neighbors_per_node(self._c_model, index.neighbors_per_node) + if index.indexing_search_count is not None: + obx_model_property_index_hnsw_indexing_search_count(self._c_model, index.indexing_search_count) + if index.flags is not None: + obx_model_property_index_hnsw_flags(self._c_model, index.flags) + if index.distance_type is not None: + obx_model_property_index_hnsw_distance_type(self._c_model, index.distance_type) + if index.reparation_backlink_probability is not None: + obx_model_property_index_hnsw_reparation_backlink_probability(self._c_model, + index.reparation_backlink_probability) + if index.vector_cache_hint_size_kb is not None: + obx_model_property_index_hnsw_vector_cache_hint_size_kb(self._c_model, index.vector_cache_hint_size_kb) + + def _create_index(self, index: Union[Index, HnswIndex]): + if isinstance(index, HnswIndex): + self._set_hnsw_params(index) + obx_model_property_index_id(self._c_model, index.id, index.uid) + + def _create_property(self, prop: Property): + obx_model_property(self._c_model, c_str(prop.name), prop._ob_type, prop.id, prop.uid) + if prop._flags != 0: + obx_model_property_flags(self._c_model, prop._flags) + if prop.index is not None: + self._create_index(prop.index) + + def _create_entity(self, entity: _Entity): + obx_model_entity(self._c_model, c_str(entity._name), entity._id, entity._uid) + for prop in entity._properties: + self._create_property(prop) + obx_model_entity_last_property_id(self._c_model, entity._last_property_iduid.id, entity._last_property_iduid.uid) + + def _create_c_model(self) -> obx_model: # Called by StoreOptions + """ Creates the OBX model by invoking the C API. + Before calling this method, IDs/UIDs must be assigned either manually or via sync_model(). """ + self._c_model = obx_model() + for entity in self.entities: + self._create_entity(entity) + if self.last_relation_iduid: + obx_model_last_relation_id(self._c_model, self.last_relation_iduid.id, self.last_relation_iduid.uid) + if self.last_index_iduid: + obx_model_last_index_id(self._c_model, self.last_index_iduid.id, self.last_index_iduid.uid) + if self.last_entity_iduid: + obx_model_last_entity_id(self._c_model, self.last_entity_iduid.id, self.last_entity_iduid.uid) + return self._c_model diff --git a/objectbox/model/properties.py b/objectbox/model/properties.py index 08810f2..e9f438b 100644 --- a/objectbox/model/properties.py +++ b/objectbox/model/properties.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 ObjectBox Ltd. All rights reserved. +# Copyright 2019-2024 ObjectBox Ltd. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,10 +13,14 @@ # limitations under the License. from enum import IntEnum - -from objectbox.c import * +from datetime import datetime import flatbuffers.number_types +import numpy as np +from dataclasses import dataclass +from objectbox.c import * +from objectbox.condition import PropertyQueryCondition, PropertyQueryConditionOp +from objectbox.model.iduid import IdUid class PropertyType(IntEnum): bool = OBXPropertyType_Bool @@ -28,9 +32,18 @@ class PropertyType(IntEnum): float = OBXPropertyType_Float double = OBXPropertyType_Double string = OBXPropertyType_String - # date = OBXPropertyType_Date + date = OBXPropertyType_Date + dateNano = OBXPropertyType_DateNano + flex = OBXPropertyType_Flex # relation = OBXPropertyType_Relation + boolVector = OBXPropertyType_BoolVector byteVector = OBXPropertyType_ByteVector + shortVector = OBXPropertyType_ShortVector + charVector = OBXPropertyType_CharVector + intVector = OBXPropertyType_IntVector + longVector = OBXPropertyType_LongVector + floatVector = OBXPropertyType_FloatVector + doubleVector = OBXPropertyType_DoubleVector # stringVector = OBXPropertyType_StringVector @@ -44,32 +57,156 @@ class PropertyType(IntEnum): PropertyType.float: flatbuffers.number_types.Float32Flags, PropertyType.double: flatbuffers.number_types.Float64Flags, PropertyType.string: flatbuffers.number_types.UOffsetTFlags, - # PropertyType.date: flatbuffers.number_types.Int64Flags, + PropertyType.date: flatbuffers.number_types.Int64Flags, + PropertyType.dateNano: flatbuffers.number_types.Int64Flags, + PropertyType.flex: flatbuffers.number_types.UOffsetTFlags, # PropertyType.relation: flatbuffers.number_types.Int64Flags, + PropertyType.boolVector: flatbuffers.number_types.UOffsetTFlags, PropertyType.byteVector: flatbuffers.number_types.UOffsetTFlags, + PropertyType.shortVector: flatbuffers.number_types.UOffsetTFlags, + PropertyType.charVector: flatbuffers.number_types.UOffsetTFlags, + PropertyType.intVector: flatbuffers.number_types.UOffsetTFlags, + PropertyType.longVector: flatbuffers.number_types.UOffsetTFlags, + PropertyType.floatVector: flatbuffers.number_types.UOffsetTFlags, + PropertyType.doubleVector: flatbuffers.number_types.UOffsetTFlags, # PropertyType.stringVector: flatbuffers.number_types.UOffsetTFlags, } +class IndexType(IntEnum): + VALUE = OBXPropertyFlags_INDEXED + HASH = OBXPropertyFlags_INDEX_HASH + HASH64 = OBXPropertyFlags_INDEX_HASH64 + + +class Index: + # TODO HNSW isn't a `type` but HASH and HASH64 are, remove type member and make HashIndex and Hash64Index classes? + + def __init__(self, type: IndexType = IndexType.VALUE, uid: int = 0): + self.type = type + + self.iduid = IdUid(0, uid) + + @property + def id(self): + return self.iduid.id + + @property + def uid(self): + return self.iduid.uid + + def has_uid(self): + return self.iduid.uid != 0 + + +class HnswFlags(IntEnum): + NONE = 0 + DEBUG_LOGS = 1 + DEBUG_LOGS_DETAILED = 2 + VECTOR_CACHE_SIMD_PADDING_OFF = 4 + REPARATION_LIMIT_CANDIDATES = 8 + + +class VectorDistanceType(IntEnum): + UNKNOWN = OBXVectorDistanceType_UNKNOWN + EUCLIDEAN = OBXVectorDistanceType_EUCLIDEAN + COSINE = OBXVectorDistanceType_COSINE + DOT_PRODUCT = OBXVectorDistanceType_DOT_PRODUCT + DOT_PRODUCT_NON_NORMALIZED = OBXVectorDistanceType_DOT_PRODUCT_NON_NORMALIZED + + +VectorDistanceType.UNKNOWN.__doc__ = "Not a real type, just best practice (e.g. forward compatibility)" +VectorDistanceType.EUCLIDEAN.__doc__ = "The default; typically 'euclidean squared' internally." +VectorDistanceType.COSINE.__doc__ = """ +Cosine similarity compares two vectors irrespective of their magnitude (compares the angle of two vectors). +Often used for document or semantic similarity. +Value range: 0.0 - 2.0 (0.0: same direction, 1.0: orthogonal, 2.0: opposite direction) +""" +VectorDistanceType.DOT_PRODUCT.__doc__ = """ +For normalized vectors (vector length == 1.0), the dot product is equivalent to the cosine similarity. +Because of this, the dot product is often preferred as it performs better. +Value range (normalized vectors): 0.0 - 2.0 (0.0: same direction, 1.0: orthogonal, 2.0: opposite direction) +""" +VectorDistanceType.DOT_PRODUCT_NON_NORMALIZED.__doc__ = """ +A custom dot product similarity measure that does not require the vectors to be normalized. +Note: this is no replacement for cosine similarity (like DotProduct for normalized vectors is). +The non-linear conversion provides a high precision over the entire float range (for the raw dot product). +The higher the dot product, the lower the distance is (the nearer the vectors are). +The more negative the dot product, the higher the distance is (the farther the vectors are). +Value range: 0.0 - 2.0 (nonlinear; 0.0: nearest, 1.0: orthogonal, 2.0: farthest) +""" + + +class HnswIndex: + def __init__(self, + dimensions: int, + neighbors_per_node: Optional[int] = None, + indexing_search_count: Optional[int] = None, + flags: HnswFlags = HnswFlags.NONE, + distance_type: VectorDistanceType = VectorDistanceType.EUCLIDEAN, + reparation_backlink_probability: Optional[float] = None, + vector_cache_hint_size_kb: Optional[float] = None, + uid: int = 0): + self.dimensions = dimensions + self.neighbors_per_node = neighbors_per_node + self.indexing_search_count = indexing_search_count + self.flags = flags + self.distance_type = distance_type + self.reparation_backlink_probability = reparation_backlink_probability + self.vector_cache_hint_size_kb = vector_cache_hint_size_kb + + self.iduid = IdUid(0, uid) + + @property + def id(self): + return self.iduid.id + + @property + def uid(self): + return self.iduid.uid + + def has_uid(self): + return self.uid != 0 + + class Property: - def __init__(self, py_type: type, id: int, uid: int, type: PropertyType = None): - self._id = id - self._uid = uid - self._name = "" # set in Entity.fill_properties() + def __init__(self, pytype: Type, uid: int = 0, **kwargs): + self.iduid = IdUid(0, uid) + self.name = "" # set in Entity.fill_properties() + self.index = kwargs.get('index', None) - self._py_type = py_type - self._ob_type = type if type != None else self.__determine_ob_type() + self._py_type = pytype + self._ob_type = kwargs['type'] if 'type' in kwargs else self._determine_ob_type() self._fb_type = fb_type_map[self._ob_type] - self._is_id = isinstance(self, Id) - self._flags = OBXPropertyFlags(0) - self.__set_flags() + self._flags = 0 + self._set_flags() + + self._fb_slot = None + self._fb_v_offset = None + + @property + def id(self): + return self.iduid.id + + @property + def uid(self): + return self.iduid.uid - # FlatBuffers marshalling information - self._fb_slot = self._id - 1 - self._fb_v_offset = 4 + 2*self._fb_slot + def has_uid(self): + return self.uid != 0 - def __determine_ob_type(self) -> OBXPropertyType: + def is_id(self) -> bool: + return isinstance(self, Id) + + def on_sync(self): + """ Method called once ID/UID are synced with the model file. """ + assert self.iduid.is_assigned() + self._fb_slot = self.id - 1 + self._fb_v_offset = 4 + 2 * self._fb_slot + + def _determine_ob_type(self) -> OBXPropertyType: + """ Tries to infer the OBX property type from the Python type. """ ts = self._py_type if ts == str: return OBXPropertyType_String @@ -77,6 +214,8 @@ def __determine_ob_type(self) -> OBXPropertyType: return OBXPropertyType_Long elif ts == bytes: # or ts == bytearray: might require further tests on read objects due to mutability return OBXPropertyType_ByteVector + elif ts == list or ts == np.ndarray: + return OBXPropertyType_DoubleVector elif ts == float: return OBXPropertyType_Double elif ts == bool: @@ -84,12 +223,275 @@ def __determine_ob_type(self) -> OBXPropertyType: else: raise Exception("unknown property type %s" % ts) - def __set_flags(self): - if self._is_id: - self._flags = OBXPropertyFlags_ID + def _set_flags(self): + if self.is_id(): + self._flags |= OBXPropertyFlags_ID + + if self.index is not None: + self._flags |= OBXPropertyFlags_INDEXED + if isinstance(self.index, Index): # Generic index + self._flags |= self.index.type + + def _assert_ids_assigned(self): + # Using assert(s) so they can be optionally disabled for performance + assert self.iduid.is_assigned(), f"Property \"{self.name}\" ID not assigned" + if self.index is not None: + assert self.index.iduid.is_assigned(), f"Property \"{self.name}\" index ID not assigned" + +class _NumericProperty(Property): + """Common class for numeric conditions. + Implicitly no support for equals/not_equals, see also _IntProperty below. + """ + def __init__(self, py_type : Type, **kwargs): + super(_NumericProperty, self).__init__(py_type, **kwargs) + + def greater_than(self, value) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.GT, args) + + def greater_or_equal(self, value) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.GTE, args) + + def less_than(self, value) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.LT, args) + + def less_or_equal(self, value) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.LTE, args) + + def between(self, a, b) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'a': a, 'b': b} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.BETWEEN, args) + +class _IntProperty(_NumericProperty): + """Integer-based conditions. + Adds support for equals/not_equals. + """ + def __init__(self, py_type : Type, **kwargs): + super(_IntProperty, self).__init__(py_type, **kwargs) + + def equals(self, value) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.EQ, args) + + def not_equals(self, value) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.NOT_EQ, args) # ID property (primary key) -class Id(Property): - def __init__(self, py_type: type = int, id: int = 0, uid: int = 0): - super(Id, self).__init__(py_type, id, uid) +class Id(_IntProperty): + def __init__(self, id : int = 0, uid : int = 0, py_type: type = int): + super(Id, self).__init__(py_type, id=id, uid=uid) + +# Bool property +class Bool(_IntProperty): + def __init__(self, id : int = 0, uid : int = 0, **kwargs): + super(Bool, self).__init__(bool, type=PropertyType.bool, id=id, uid=uid, **kwargs) + +# String property with starts/ends_with +class String(Property): + def __init__(self, id: int = 0, uid : int = 0, **kwargs): + super(String, self).__init__(str, type=PropertyType.string, id=id, uid=uid, **kwargs) + + def starts_with(self, value: str, case_sensitive: bool = True) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value, 'case_sensitive': case_sensitive} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.STARTS_WITH, args) + + def ends_with(self, value: str, case_sensitive: bool = True) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value, 'case_sensitive': case_sensitive} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.ENDS_WITH, args) + + def equals(self, value, case_sensitive: bool = True) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value, 'case_sensitive': case_sensitive} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.EQ, args) + + def not_equals(self, value, case_sensitive: bool = True) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value, 'case_sensitive': case_sensitive} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.NOT_EQ, args) + + def contains(self, value: str, case_sensitive: bool = True) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value, 'case_sensitive': case_sensitive} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.CONTAINS, args) + + def greater_than(self, value, case_sensitive: bool = True) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value, 'case_sensitive': case_sensitive} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.GT, args) + + def greater_or_equal(self, value, case_sensitive: bool = True) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value, 'case_sensitive': case_sensitive} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.GTE, args) + + def less_than(self, value, case_sensitive: bool = True) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value, 'case_sensitive': case_sensitive} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.LT, args) + + def less_or_equal(self, value, case_sensitive: bool = True) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value, 'case_sensitive': case_sensitive} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.LTE, args) + + + +# Signed Integer Numeric Properties +class Int8(_IntProperty): + def __init__(self, id : int = 0, uid : int = 0, **kwargs): + super(Int8, self).__init__(int, type=PropertyType.byte, id=id, uid=uid, **kwargs) +class Int16(_IntProperty): + def __init__(self, id : int = 0, uid : int = 0, **kwargs): + super(Int16, self).__init__(int, type=PropertyType.short, id=id, uid=uid, **kwargs) +class Int32(_IntProperty): + def __init__(self, id : int = 0, uid : int = 0, **kwargs): + super(Int32, self).__init__(int, type=PropertyType.int, id=id, uid=uid, **kwargs) +class Int64(_IntProperty): + def __init__(self, id : int = 0, uid : int = 0, **kwargs): + super(Int64, self).__init__(int, type=PropertyType.long, id=id, uid=uid, **kwargs) + +# Floating-Point Numeric Properties +class Float32(_NumericProperty): + def __init__(self, id : int = 0, uid : int = 0, **kwargs): + super(Float32, self).__init__(float, type=PropertyType.float, id=id, uid=uid, **kwargs) + +class Float64(_NumericProperty): + def __init__(self, id : int = 0, uid : int = 0, **kwargs): + super(Float64, self).__init__(float, type=PropertyType.double, id=id, uid=uid, **kwargs) + +# Date Properties +class Date(_IntProperty): + def __init__(self, py_type = datetime, id : int = 0, uid : int = 0, **kwargs): + super(Date, self).__init__(py_type, type=PropertyType.date, id=id, uid=uid, **kwargs) + +class DateNano(_IntProperty): + def __init__(self, py_type = datetime, id : int = 0, uid : int = 0, **kwargs): + super(DateNano, self).__init__(py_type, type=PropertyType.dateNano, id=id, uid=uid, **kwargs) + +# Bytes Property +class Bytes(_NumericProperty): + def __init__(self, id: int = 0, uid : int = 0, **kwargs): + super(Bytes, self).__init__(bytes, type=PropertyType.byteVector, id=id, uid=uid, **kwargs) + + def equals(self, value) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.EQ, args) + + def greater_than(self, value) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.GT, args) + + def greater_or_equal(self, value) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.GTE, args) + + def less_than(self, value) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.LT, args) + + def less_or_equal(self, value) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'value': value} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.LTE, args) + +# Flex Property +class Flex(Property): + def __init__(self, id : int = 0, uid : int = 0, **kwargs): + super(Flex, self).__init__(Generic, type=PropertyType.flex, id=id, uid=uid, **kwargs) + def contains_key_value(self, key: str, value: str, case_sensitive: bool = True) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'key': key, 'value': value, 'case_sensitive': case_sensitive} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.CONTAINS_KEY_VALUE, args) + +class _VectorProperty(Property): + def __init__(self, py_type : Type, **kwargs): + super(_VectorProperty, self).__init__(py_type, **kwargs) + +class BoolVector(_VectorProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(BoolVector, self).__init__(np.ndarray, type=PropertyType.boolVector, id=id, uid=uid, **kwargs) +class Int8Vector(_VectorProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(Int8Vector, self).__init__(bytes, type=PropertyType.byteVector, id=id, uid=uid, **kwargs) + +class Int16Vector(_VectorProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(Int16Vector, self).__init__(np.ndarray, type=PropertyType.shortVector, id=id, uid=uid, **kwargs) + +class CharVector(_VectorProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(CharVector, self).__init__(np.ndarray, type=PropertyType.charVector, id=id, uid=uid, **kwargs) + +class Int32Vector(_VectorProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(Int32Vector, self).__init__(np.ndarray, type=PropertyType.intVector, id=id, uid=uid, **kwargs) + +class Int64Vector(_VectorProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(Int64Vector, self).__init__(np.ndarray, type=PropertyType.longVector, id=id, uid=uid, **kwargs) + +class Float32Vector(_VectorProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(Float32Vector, self).__init__(np.ndarray, type=PropertyType.floatVector, id=id, uid=uid, **kwargs) + def nearest_neighbor(self, query_vector, element_count: int) -> PropertyQueryCondition: + self._assert_ids_assigned() + args = {'query_vector': query_vector, 'element_count': element_count} + return PropertyQueryCondition(self.id, PropertyQueryConditionOp.NEAREST_NEIGHBOR, args) + +class Float64Vector(_VectorProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(Float64Vector, self).__init__(np.ndarray, type=PropertyType.doubleVector, id=id, uid=uid, **kwargs) + +class _ListProperty(Property): + def __init__(self, **kwargs): + super(_ListProperty, self).__init__(list, **kwargs) + +class BoolList(_ListProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(BoolList, self).__init__(type=PropertyType.boolVector, id=id, uid=uid, **kwargs) + +class Int8List(_ListProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(Int8List, self).__init__(type=PropertyType.byteVector, id=id, uid=uid, **kwargs) + +class Int16List(_ListProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(Int16List, self).__init__(type=PropertyType.shortVector, id=id, uid=uid, **kwargs) + +class Int32List(_ListProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(Int32List, self).__init__(type=PropertyType.intVector, id=id, uid=uid, **kwargs) + +class Int64List(_ListProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(Int64List, self).__init__(type=PropertyType.longVector, id=id, uid=uid, **kwargs) + +class Float32List(_ListProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(Float32List, self).__init__(type=PropertyType.floatVector, id=id, uid=uid, **kwargs) + +class Float64List(_ListProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(Float64List, self).__init__(type=PropertyType.doubleVector, id=id, uid=uid, **kwargs) + +class CharList(_ListProperty): + def __init__(self, id: int = 0, uid: int = 0, **kwargs): + super(CharList, self).__init__(type=PropertyType.charVector, id=id, uid=uid, **kwargs) diff --git a/objectbox/objectbox.py b/objectbox/objectbox.py index 1a0be39..1a6dff7 100644 --- a/objectbox/objectbox.py +++ b/objectbox/objectbox.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 ObjectBox Ltd. All rights reserved. +# Copyright 2019-2024 ObjectBox Ltd. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,19 +13,11 @@ # limitations under the License. -from objectbox.c import * -import objectbox.transaction +import objectbox.store +from warnings import warn - -class ObjectBox: - def __init__(self, c_store: OBX_store_p): - self._c_store = c_store - - def __del__(self): - obx_store_close(self._c_store) - - def read_tx(self): - return objectbox.transaction.read(self) - - def write_tx(self): - return objectbox.transaction.write(self) +class ObjectBox(objectbox.store.Store): + def __init__(self, c_store): + """This throws a deprecation warning on initialization.""" + warn(f'{self.__class__.__name__} will be deprecated, use Store from objectbox.store.', DeprecationWarning, stacklevel=2) + super().__init__(c_store=c_store) diff --git a/objectbox/query.py b/objectbox/query.py new file mode 100644 index 0000000..8b1792e --- /dev/null +++ b/objectbox/query.py @@ -0,0 +1,174 @@ +# Copyright 2019-2024 ObjectBox Ltd. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from objectbox.c import * + + +class Query: + def __init__(self, c_query, box: 'Box'): + self._c_query = c_query + self._box = box + self._entity = self._box._entity + self._store = box._store + + def find(self) -> list: + """ Finds a list of objects matching query. """ + with self._store.read_tx(): # We need a read transaction to ensure the object data stays valid + # OBX_bytes_array* + c_bytes_array_p = obx_query_find(self._c_query) + try: + # OBX_bytes_array + c_bytes_array = c_bytes_array_p.contents + + result = [] + for i in range(c_bytes_array.count): + # OBX_bytes + c_bytes = c_bytes_array.data[i] + data = c_voidp_as_bytes(c_bytes.data, c_bytes.size) + result.append(self._box._entity._unmarshal(data)) + return result + finally: + obx_bytes_array_free(c_bytes_array_p) + + def find_ids(self) -> List[int]: + """ Finds a list of object IDs matching query. The result is sorted by ID (ascending order). """ + c_id_array_p = obx_query_find_ids(self._c_query) + try: + c_id_array: OBX_id_array = c_id_array_p.contents + if c_id_array.count == 0: + return [] + ids = ctypes.cast(c_id_array.ids, ctypes.POINTER(obx_id * c_id_array.count)) + return list(ids.contents) + finally: + obx_id_array_free(c_id_array_p) + + def find_with_scores(self): + """ Finds objects matching the query associated to their query score (e.g. distance in NN search). + The result is sorted by score in ascending order. """ + with self._store.read_tx(): # We need a read transaction to ensure the object data stays valid + c_bytes_score_array_p = obx_query_find_with_scores(self._c_query) + try: + # OBX_bytes_score_array + c_bytes_score_array: OBX_bytes_score_array = c_bytes_score_array_p.contents + result = [] + for i in range(c_bytes_score_array.count): + c_bytes_score: OBX_bytes_score = c_bytes_score_array.bytes_scores[i] + data = c_voidp_as_bytes(c_bytes_score.data, c_bytes_score.size) + score = c_bytes_score.score + + object_ = self._box._entity._unmarshal(data) + result.append((object_, score)) + return result + finally: + obx_bytes_score_array_free(c_bytes_score_array_p) + + def find_ids_with_scores(self) -> List[Tuple[int, float]]: + """ Finds object IDs matching the query associated to their query score (e.g. distance in NN search). + The resulting list is sorted by score in ascending order. """ + c_id_score_array_p = obx_query_find_ids_with_scores(self._c_query) + try: + # OBX_id_score_array + c_id_score_array: OBX_bytes_score_array = c_id_score_array_p.contents + result = [] + for i in range(c_id_score_array.count): + c_id_score: OBX_id_score = c_id_score_array.ids_scores[i] + result.append((c_id_score.id, c_id_score.score)) + return result + finally: + obx_id_score_array_free(c_id_score_array_p) + + def find_ids_by_score(self) -> List[int]: + """ Finds object IDs matching the query ordered by their query score (e.g. distance in NN search). + The resulting list of IDs is sorted by score in ascending order. """ + # TODO extract utility function for ID array conversion + c_id_array_p = obx_query_find_ids_by_score(self._c_query) + try: + c_id_array: OBX_id_array = c_id_array_p.contents + if c_id_array.count == 0: + return [] + ids = ctypes.cast(c_id_array.ids, ctypes.POINTER(obx_id * c_id_array.count)) + return list(ids.contents) + finally: + obx_id_array_free(c_id_array_p) + + def find_ids_by_score_numpy(self) -> np.array: + """ Finds object IDs matching the query ordered by their query score (e.g. distance in NN search). + The resulting list of IDs is sorted by score in ascending order. """ + # TODO extract utility function for ID array conversion + c_id_array_p = obx_query_find_ids_by_score(self._c_query) + try: + c_id_array: OBX_id_array = c_id_array_p.contents + c_count = c_id_array.count + numpy_array = np.empty(c_count, dtype=np.uint64) + if c_count > 0: + c_ids = ctypes.cast(c_id_array.ids, ctypes.POINTER(obx_id)) + ctypes.memmove(numpy_array.ctypes.data, c_ids, numpy_array.nbytes) + return numpy_array + finally: + obx_id_array_free(c_id_array_p) + + def count(self) -> int: + count = ctypes.c_uint64() + obx_query_count(self._c_query, ctypes.byref(count)) + return int(count.value) + + def remove(self) -> int: + count = ctypes.c_uint64() + obx_query_remove(self._c_query, ctypes.byref(count)) + return int(count.value) + + def offset(self, offset: int) -> 'Query': + obx_query_offset(self._c_query, offset) + return self + + def limit(self, limit: int) -> 'Query': + obx_query_limit(self._c_query, limit) + return self + + def set_parameter_string(self, prop: Union[int, str, 'Property'], value: str) -> 'Query': + prop_id = self._entity._get_property_id(prop) + obx_query_param_string(self._c_query, self._entity._id, prop_id, c_str(value)) + return self + + def set_parameter_int(self, prop: Union[int, str, 'Property'], value: int) -> 'Query': + prop_id = self._entity._get_property_id(prop) + obx_query_param_int(self._c_query, self._entity._id, prop_id, value) + return self + + def set_parameter_vector_f32(self, + prop: Union[int, str, 'Property'], + value: Union[List[float], np.ndarray]) -> 'Query': + if isinstance(value, np.ndarray) and value.dtype != np.float32: + raise Exception(f"value dtype is expected to be np.float32, got: {value.dtype}") + prop_id = self._entity._get_property_id(prop) + c_value = c_array(value, ctypes.c_float) + num_el = len(value) + obx_query_param_vector_float32(self._c_query, self._entity._id, prop_id, c_value, num_el) + return self + + def offset(self, offset: int): + return obx_query_offset(self._c_query, offset) + + def limit(self, limit: int): + return obx_query_limit(self._c_query, limit) + + def set_parameter_alias_string(self, alias: str, value: str): + return obx_query_param_alias_string(self._c_query, c_str(alias), c_str(value)) + + def set_parameter_alias_int(self, alias: str, value: int): + return obx_query_param_alias_int(self._c_query, c_str(alias), value) + + def set_parameter_alias_vector_f32(self, alias: str, value: Union[List[float], np.ndarray]): + return obx_query_param_alias_vector_float32(self._c_query, c_str(alias), c_array(value, ctypes.c_float), + len(value)) diff --git a/objectbox/query_builder.py b/objectbox/query_builder.py new file mode 100644 index 0000000..2147fcc --- /dev/null +++ b/objectbox/query_builder.py @@ -0,0 +1,200 @@ +import ctypes +import numpy as np +from typing import * + +from objectbox.c import * +from objectbox.model.properties import Property +from objectbox.store import Store +from objectbox.query import Query +from objectbox.utils import check_float_vector + + +class QueryBuilder: + def __init__(self, store: Store, box: 'Box'): + self._box = box + self._entity = box._entity + self._c_builder = obx_query_builder(store._c_store, box._entity._id) + + def close(self) -> int: + return obx_qb_close(self._c_builder) + + def error_code(self) -> int: + return obx_qb_error_code(self._c_builder) + + def error_message(self) -> str: + return obx_qb_error_message(self._c_builder) + + def equals_string(self, prop: Union[int, str, Property], value: str, case_sensitive: bool = True) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_equals_string(self._c_builder, prop_id, c_str(value), case_sensitive) + return cond + + def not_equals_string(self, prop: Union[int, str, Property], value: str, + case_sensitive: bool = True) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_not_equals_string(self._c_builder, prop_id, c_str(value), case_sensitive) + return cond + + def contains_string(self, prop: Union[int, str, Property], value: str, case_sensitive: bool = True) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_contains_string(self._c_builder, prop_id, c_str(value), case_sensitive) + return cond + + def starts_with_string(self, prop: Union[int, str, Property], value: str, + case_sensitive: bool = True) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_starts_with_string(self._c_builder, prop_id, c_str(value), case_sensitive) + return cond + + def ends_with_string(self, prop: Union[int, str, Property], value: str, case_sensitive: bool = True) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_ends_with_string(self._c_builder, prop_id, c_str(value), case_sensitive) + return cond + + def greater_than_string(self, prop: Union[int, str, Property], value: str, + case_sensitive: bool = True) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_greater_than_string(self._c_builder, prop_id, c_str(value), case_sensitive) + return cond + + def greater_or_equal_string(self, prop: Union[int, str, Property], value: str, + case_sensitive: bool = True) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_greater_or_equal_string(self._c_builder, prop_id, c_str(value), case_sensitive) + return cond + + def less_than_string(self, prop: Union[int, str, Property], value: str, case_sensitive: bool = True) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_less_than_string(self._c_builder, prop_id, c_str(value), case_sensitive) + return cond + + def less_or_equal_string(self, prop: Union[int, str, Property], value: str, + case_sensitive: bool = True) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_less_or_equal_string(self._c_builder, prop_id, c_str(value), case_sensitive) + return cond + + def equals_int(self, prop: Union[int, str, Property], value: int) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_equals_int(self._c_builder, prop_id, value) + return cond + + def not_equals_int(self, prop: Union[int, str, Property], value: int) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_not_equals_int(self._c_builder, prop_id, value) + return cond + + def greater_than_int(self, prop: Union[int, str, Property], value: int) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_greater_than_int(self._c_builder, prop_id, value) + return cond + + def greater_than_double(self, prop: Union[int, str, Property], value: float) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_greater_than_double(self._c_builder, prop_id, value) + return cond + + def greater_or_equal_int(self, prop: Union[int, str, Property], value: int) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_greater_or_equal_int(self._c_builder, prop_id, value) + return cond + + def greater_or_equal_double(self, prop: Union[int, str, Property], value: float) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_greater_or_equal_double(self._c_builder, prop_id, value) + return cond + + def less_than_int(self, prop: Union[int, str, Property], value: int) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_less_than_int(self._c_builder, prop_id, value) + return cond + + def less_than_double(self, prop: Union[int, str, Property], value: float) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_less_than_double(self._c_builder, prop_id, value) + return cond + + def less_or_equal_int(self, prop: Union[int, str, Property], value: int) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_less_or_equal_int(self._c_builder, prop_id, value) + return cond + + def less_or_equal_double(self, prop: Union[int, str, Property], value: float) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_less_or_equal_double(self._c_builder, prop_id, value) + return cond + + def between_2ints(self, prop: Union[int, str, Property], value_a: int, value_b: int) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_between_2ints(self._c_builder, prop_id, value_a, value_b) + return cond + + def between_2doubles(self, prop: Union[int, str, Property], value_a: float, value_b: float) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_between_2doubles(self._c_builder, prop_id, value_a, value_b) + return cond + + def equals_bytes(self, prop: Union[int, str, Property], value: bytes) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_equals_bytes(self._c_builder, prop_id, value, len(value)) + return cond + + def greater_than_bytes(self, prop: Union[int, str, Property], value: bytes) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_greater_than_bytes(self._c_builder, prop_id, value, len(value)) + return cond + + def greater_or_equal_bytes(self, prop: Union[int, str, Property], value: bytes) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_greater_or_equal_bytes(self._c_builder, prop_id, value, len(value)) + return cond + + def less_than_bytes(self, prop: Union[int, str, Property], value: bytes) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_less_than_bytes(self._c_builder, prop_id, value, len(value)) + return cond + + def less_or_equal_bytes(self, prop: Union[int, str, Property], value: bytes) -> obx_qb_cond: + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_less_or_equal_bytes(self._c_builder, prop_id, value, len(value)) + return cond + + def nearest_neighbors_f32(self, + prop: Union[int, str, Property], + query_vector: Union[np.ndarray, List[float]], + element_count: int): + check_float_vector(query_vector, "query_vector") + prop_id = self._entity._get_property_id(prop) + c_query_vector = c_array(query_vector, ctypes.c_float) + cond = obx_qb_nearest_neighbors_f32(self._c_builder, prop_id, c_query_vector, element_count) + return cond + + def contains_key_value(self, prop: Union[int, str, Property], key: str, value: str, + case_sensitive: bool = True) -> obx_qb_cond: + """ Checks whether the given Flex property, interpreted as a dictionary and indexed at key, has a value + corresponding to the given value. + + :param case_sensitive: + If false, ignore case when matching value + """ + prop_id = self._entity._get_property_id(prop) + cond = obx_qb_contains_key_value_string(self._c_builder, prop_id, c_str(key), c_str(value), case_sensitive) + return cond + + def any(self, conditions: List[obx_qb_cond]) -> obx_qb_cond: + c_conditions = c_array(conditions, obx_qb_cond) + cond = obx_qb_any(self._c_builder, c_conditions, len(conditions)) + return cond + + def all(self, conditions: List[obx_qb_cond]) -> obx_qb_cond: + c_conditions = c_array_pointer(conditions, obx_qb_cond) + cond = obx_qb_all(self._c_builder, c_conditions, len(conditions)) + return cond + + def build(self) -> Query: + c_query = obx_query(self._c_builder) + return Query(c_query, self._box) + + def alias(self, alias: str): + obx_qb_param_alias(self._c_builder, c_str(alias)) + return self diff --git a/objectbox/store.py b/objectbox/store.py new file mode 100644 index 0000000..de9d109 --- /dev/null +++ b/objectbox/store.py @@ -0,0 +1,271 @@ +# Copyright 2019-2024 ObjectBox Ltd. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import inspect +import logging +import os +import sys +from types import ModuleType + +import objectbox.c as c +import objectbox.transaction +from objectbox.model.idsync import sync_model +from objectbox.store_options import StoreOptions +import objectbox +from objectbox.model.entity import _Entity +from objectbox.model.model import Model +from typing import * + + +class Store: + def __init__(self, + model: Optional[Union[Model, str]] = "default", + model_json_file: Optional[str] = None, + directory: Optional[str] = None, + max_db_size_in_kb: Optional[int] = None, + max_data_size_in_kb: Optional[int] = None, + file_mode: Optional[int] = None, + max_readers: Optional[int] = None, + no_reader_thread_locals: Optional[bool] = None, + model_bytes: Optional[bytes] = None, + model_bytes_direct: Optional[bytes] = None, + read_schema: Optional[bool] = None, + use_previous_commit: Optional[bool] = None, + read_only: Optional[bool] = None, + debug_flags: Optional[c.DebugFlags] = None, + async_max_queue_length: Optional[int] = None, + async_throttle_at_queue_length: Optional[int] = None, + async_throttle_micros: Optional[int] = None, + async_max_in_tx_duration: Optional[int] = None, + async_max_in_tx_operations: Optional[int] = None, + async_pre_txn_delay: Optional[int] = None, + async_post_txn_delay: Optional[int] = None, + async_minor_refill_threshold: Optional[int] = None, + async_minor_refill_max_count: Optional[int] = None, + async_object_bytes_max_cache_size: Optional[int] = None, + async_object_bytes_max_size_to_cache: Optional[int] = None, + c_store: Optional[c.OBX_store_p] = None): + + """Opens an ObjectBox database Store + + :param model: + Database schema model. + :param directory: + Store directory. Defaults to "objectbox". + Use prefix "memory:" to open an in-memory database, e.g. "memory:myapp" + :param max_db_size_in_kb: + Maximum database size. Defaults to one gigabyte. + :param max_data_size_in_kb: + Maximum data in database size tracking. Defaults to being disabled. + This is a more involved size tacking. + Recommended only if stricter accurate limit is required. + Data size must be below database size. + :param file_mode: + Unix-style file mode options. Defaults to "int('644',8)". + This option is ignored on Windows platforms. + :param max_readers: + Maximum number of readers (related to read transactions). + Default value (currently 126) is suitable for most applications. + :param no_reader_thread_locals: + Disables the usage of thread locals for "readers" related to read transactions. + This can make sense if you are using a lot of threads that are kept alive. + :param model_bytes: + Database schema model given by flatbuffers bytes serialized model. + :param model_bytes_direct: + Database schema model given by flatbuffers bytes serialized model without copying. + :param read_schema: + Advanced settings. + :param use_previous_commit: + Advanced setting recommended to set with read_only to ensure no data is lost. + :param read_only: + Open store in read-only mode: no schema update, no write transactions. Defaults to false. + :param debug_flags: + Set debug flags. Defaults to DebugFlags.NONE. + :param async_max_queue_length: + Maximum size of the queue before new transactions will be rejected. + :param async_throttle_at_queue_length: + Throttle queue submitter when hitting this water mark. + :param async_throttle_micros: + Sleeping time for throttled queue submitter. + :param async_max_in_tx_duration: + Maximum duration spent in a transaction before queue enforces a commit. + :param async_max_in_tx_operations: + Maximum number of operations performed in a transaction before queye enforces a commit. + :param async_pre_txn_delay: + Delay (in micro seconds) before queue is triggered by new element. + :param async_post_txn_delay: + Delay (in micro seconds) after a transaction was committed. + :param async_minor_refill_threshold: + Number of operations to be considered a "minor refill". + :param async_minor_refill_max_count: + If set, allows "minor refills" with small batches that came in (off by default). + :param async_object_bytes_max_cache_size: + Total cache size. Defaults to 0.5 mega bytes. + :param async_object_bytes_max_size_to_cache: + Maximum size for an object to be cached. + :param c_store: + Internal parameter for deprecated ObjectBox interface. Do not use it; other options would be ignored if passed. + """ + + self._c_store = None + if not c_store: + options = StoreOptions() + try: + if model is not None: + model = Store._sync_model(model, model_json_file) + options.model(model) + if directory is not None: + options.directory(directory) + if max_db_size_in_kb is not None: + options.max_db_size_in_kb(max_db_size_in_kb) + if max_data_size_in_kb is not None: + options.max_data_size_in_kb(max_data_size_in_kb) + if file_mode is not None: + options.file_mode(file_mode) + if max_readers is not None: + options.max_readers(max_readers) + if no_reader_thread_locals is not None: + options.no_reader_thread_locals(no_reader_thread_locals) + if model_bytes is not None: + options.model_bytes(model_bytes) + if model_bytes_direct is not None: + options.model_bytes_direct(model_bytes_direct) + if read_schema is not None: + options.read_schema(read_schema) + if use_previous_commit is not None: + options.use_previous_commit(use_previous_commit) + if read_only is not None: + options.read_only(read_only) + if debug_flags is not None: + options.debug_flags(debug_flags) + if async_max_queue_length is not None: + options.async_max_queue_length(async_max_queue_length) + if async_throttle_at_queue_length is not None: + options.async_throttle_at_queue_length(async_throttle_at_queue_length) + if async_throttle_micros is not None: + options.async_throttle_micros(async_throttle_micros) + if async_max_in_tx_duration is not None: + options.async_max_in_tx_duration(async_max_in_tx_duration) + if async_max_in_tx_operations is not None: + options.async_max_in_tx_operations(async_max_in_tx_operations) + if async_pre_txn_delay is not None: + options.async_pre_txn_delay(async_pre_txn_delay) + if async_post_txn_delay is not None: + options.async_post_txn_delay(async_post_txn_delay) + if async_minor_refill_threshold is not None: + options.async_minor_refill_threshold(async_minor_refill_threshold) + if async_minor_refill_max_count is not None: + options.async_minor_refill_max_count(async_minor_refill_max_count) + if async_object_bytes_max_cache_size is not None: + options.async_object_bytes_max_cache_size(async_object_bytes_max_cache_size) + if async_object_bytes_max_size_to_cache is not None: + options.async_object_bytes_max_size_to_cache(async_object_bytes_max_size_to_cache) + + except c.CoreException: + options._free() + raise + self._c_store = c.obx_store_open(options._c_handle) + else: + self._c_store = c_store + + @staticmethod + def _sync_model(model: Optional[Union[Model, str]], + model_json_file: Optional[str]) -> Model: + if isinstance(model, str): # Model name provided; get entities collected via @Entity + metadata_set = objectbox.model.entity.obx_models_by_name.get(model) + if metadata_set is None: + raise ValueError( + f"Model \"{model}\" not found; ensure to set the name attribute on the model class.") + model = Model() + for metadata in metadata_set: + model.entity(metadata) + elif not isinstance(model, Model): + raise ValueError("Model must be a Model object or a string.") + + if not model_json_file: + model_json_file = Store._locate_model_json_file() + + sync_model(model, model_json_file) + + return model + + @staticmethod + def _locate_model_json_file(): + def get_module_path(module: Optional[ModuleType]) -> Optional[str]: + if module and hasattr(module, "__file__"): + return os.path.dirname(os.path.realpath(module.__file__)) + return None + + def json_file_inside_module_path(module: Optional[ModuleType]) -> Optional[str]: + module_path = get_module_path(module) + if module_path: + logging.info("Using module path to locate objectbox-model.json: ", module_path) + return os.path.join(module_path, "objectbox-model.json") + return None + + # The (direct) calling module seems like a good first choice + this_module = sys.modules[__name__] + this_module_path = get_module_path(this_module) + stack = inspect.stack() + calling_module: Optional[ModuleType] = None + for stack_element in stack: + module = inspect.getmodule(stack_element[0]) + if module is not this_module: + path = get_module_path(module) + if not path: # Cannot get the direct caller's path, so do not try further + break + if path != this_module_path: # Not inside the objectbox package + calling_module = module + break + model_json_file = json_file_inside_module_path(calling_module) + + if not model_json_file: + # Note: the main module seems less reliable, + # e.g. it resulted in a some pycharm dir when running tests from PyCharm. + model_json_file = json_file_inside_module_path(sys.modules.get('__main__')) + if not model_json_file: + model_json_file = "objectbox-model.json" + return model_json_file + + def __del__(self): + self.close() + + def box(self, entity: _Entity) -> 'objectbox.Box': + """ + Open a box for an entity. + + :param entity: + Entity type of the model + """ + return objectbox.Box(self, entity) + + def read_tx(self): + return objectbox.transaction.read(self) + + def write_tx(self): + return objectbox.transaction.write(self) + + def close(self): + c_store_to_close = self._c_store + if c_store_to_close: + self._c_store = None + c.obx_store_close(c_store_to_close) + + @staticmethod + def remove_db_files(db_dir: str): + """ Remove database files. + + :param db_dir: + Path to DB directory. + """ + c.obx_remove_db_files(c.c_str(db_dir)) diff --git a/objectbox/store_options.py b/objectbox/store_options.py new file mode 100644 index 0000000..c929682 --- /dev/null +++ b/objectbox/store_options.py @@ -0,0 +1,131 @@ +from objectbox.c import * +from objectbox.model import Model + + +class StoreOptions: + """ A RAII wrapper to the C API for setting the store options. """ + + _c_handle: Optional[int] + + def __init__(self): + self._c_handle = obx_opt() + + def _free(self): + if self._c_handle is not None: + obx_opt_free(self._c_handle) + self._c_handle = None + + def directory(self, path: str): + obx_opt_directory(self._c_handle, c_str(path)) + + def max_db_size_in_kb(self, size_in_kb: int): + obx_opt_max_db_size_in_kb(self._c_handle, size_in_kb) + + def max_data_size_in_kb(self, size_in_kb: int): + obx_opt_max_data_size_in_kb(self._c_handle, size_in_kb) + + def file_mode(self, file_mode: int): + obx_opt_file_mode(self._c_handle, file_mode) + + def max_readers(self, max_readers: int): + obx_opt_max_readers(self._c_handle, max_readers) + + def no_reader_thread_locals(self, flag: bool): + obx_opt_no_reader_thread_locals(self._c_handle, flag) + + def model(self, model: Model): + model.validate_ids_assigned() + c_model = model._create_c_model() + obx_opt_model(self._c_handle, c_model) + + def model_bytes(self, bytes_: bytes): + obx_opt_model_bytes(self._c_handle, len(bytes_)) + + def model_bytes_direct(self, bytes_: bytes): + obx_opt_model_bytes_direct(self._c_handle, len(bytes_)) + + def validate_on_open_pages(self, page_limit: int, flags: int): + raise NotImplementedError # TODO + + def validate_on_open_kv(self, flags: OBXValidateOnOpenKvFlags): + raise NotImplementedError # TODO + + def put_padding_mode(self, mode: OBXPutPaddingMode): + obx_opt_put_padding_mode(self._c_handle, mode) + + def read_schema(self, value: bool): + obx_opt_read_schema(self._c_handle, value) + + def use_previous_commit(self, value: bool): + obx_opt_use_previous_commit(self._c_handle, value) + + def read_only(self, value: bool): + obx_opt_read_only(self._c_handle, value) + + def debug_flags(self, flags: OBXDebugFlags): + obx_opt_debug_flags(self._c_handle, flags) + + def add_debug_flags(self, flags: OBXDebugFlags): + obx_opt_add_debug_flags(self._c_handle, flags) + + def async_max_queue_length(self, value: int): + obx_opt_async_max_queue_length(self._c_handle, value) + + def async_throttle_at_queue_length(self, value: int): + obx_opt_async_throttle_at_queue_length(self._c_handle, value) + + def async_throttle_micros(self, value: int): + obx_opt_async_throttle_micros(self._c_handle, value) + + def async_max_in_tx_duration(self, micros: int): + obx_opt_async_max_in_tx_duration(self._c_handle, micros) + + def async_max_in_tx_operations(self, value: int): + obx_opt_async_max_in_tx_operations(self._c_handle, value) + + def async_pre_txn_delay(self, delay_micros: int): + obx_opt_async_pre_txn_delay(self._c_handle, delay_micros) + + def async_pre_txn_delay4(self, delay_micros: int, delay2_micros: int, min_queue_length_for_delay2: int): + obx_opt_async_pre_txn_delay4(self._c_handle, delay_micros, delay2_micros, min_queue_length_for_delay2) + + def async_post_txn_delay(self, delay_micros: int): + obx_opt_async_post_txn_delay(self._c_handle, delay_micros) + + def async_post_txn_delay5(self, delay_micros: int, delay2_micros: int, min_queue_length_for_delay2: int, + subtract_processing_time: bool): + obx_opt_async_post_txn_delay5(self._c_handle, delay_micros, delay2_micros, min_queue_length_for_delay2, + subtract_processing_time) + + def async_minor_refill_threshold(self, queue_length: int): + obx_opt_async_minor_refill_threshold(self._c_handle, queue_length) + + def async_minor_refill_max_count(self, value: int): + obx_opt_async_minor_refill_max_count(self._c_handle, value) + + def async_max_tx_pool_size(self, value: int): + obx_opt_async_max_tx_pool_size(self._c_handle, value) + + def async_object_bytes_max_cache_size(self, value: int): + obx_opt_async_object_bytes_max_cache_size(self._c_handle, value) + + def async_object_bytes_max_size_to_cache(self, value: int): + obx_opt_async_object_bytes_max_size_to_cache(self._c_handle, value) + + # TODO def log_callback(self): + + def backup_restore(self, backup_file: str, flags: OBXBackupRestoreFlags): + raise NotImplementedError # TODO + + def get_directory(self) -> str: + dir_bytes = obx_opt_get_directory(self._c_handle) + return dir_bytes.decode('utf-8') + + def get_max_db_size_in_kb(self) -> int: + return obx_opt_get_max_db_size_in_kb(self._c_handle) + + def get_max_data_size_in_kb(self) -> int: + return obx_opt_get_max_data_size_in_kb(self._c_handle) + + def get_debug_flags(self) -> OBXDebugFlags: + return obx_opt_get_debug_flags(self._c_handle) diff --git a/objectbox/transaction.py b/objectbox/transaction.py index 4bd86b9..35668ca 100644 --- a/objectbox/transaction.py +++ b/objectbox/transaction.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 ObjectBox Ltd. All rights reserved. +# Copyright 2019-2024 ObjectBox Ltd. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,8 +17,8 @@ @contextmanager -def read(ob: 'ObjectBox'): - tx = obx_txn_read(ob._c_store) +def read(store: 'Store'): + tx = obx_txn_read(store._c_store) try: yield finally: @@ -26,12 +26,11 @@ def read(ob: 'ObjectBox'): @contextmanager -def write(ob: 'ObjectBox'): - tx = obx_txn_write(ob._c_store) +def write(store: 'Store'): + tx = obx_txn_write(store._c_store) try: yield obx_txn_success(tx) except: - obx_txn_abort(tx) obx_txn_close(tx) raise diff --git a/objectbox/utils.py b/objectbox/utils.py new file mode 100644 index 0000000..c2a708d --- /dev/null +++ b/objectbox/utils.py @@ -0,0 +1,57 @@ +import numpy as np + +from objectbox.c import * +from objectbox.model.properties import VectorDistanceType +from datetime import datetime, timezone + +def check_float_vector(vector: Union[np.ndarray, List[float]], vector_name: str): + """ Checks that the given vector is a float vector (either np.ndarray or Python's list). """ + if isinstance(vector, np.ndarray) and vector.dtype != np.float32: + raise Exception(f"{vector_name} dtype is expected to be np.float32, got: {vector.dtype}") + elif isinstance(vector, list) and len(vector) > 0 and (type(vector[0]) is not float): + raise Exception(f"{vector_name} is expected to be a float list, got vector[0]: {type(vector[0])}") + + +def vector_distance_f32(distance_type: VectorDistanceType, + vector1: Union[np.ndarray, List[float]], + vector2: Union[np.ndarray, List[float]], + dimension: int) -> float: + """ Utility function to calculate the distance of two vectors. """ + check_float_vector(vector1, "vector1") + check_float_vector(vector2, "vector2") + return obx_vector_distance_float32(distance_type, + c_array(vector1, ctypes.c_float), + c_array(vector2, ctypes.c_float), + dimension) + + +def vector_distance_to_relevance(distance_type: VectorDistanceType, distance: float) -> float: + """ Converts the given distance to a relevance score in range [0.0, 1.0], according to its type. """ + return obx_vector_distance_to_relevance(distance_type, distance) + +def date_value_to_int(value, multiplier: int) -> int: + if isinstance(value, datetime): + try: + return round(value.timestamp() * multiplier) # timestamp returns seconds + except OSError: + # On Windows, timestamp() raises an OSError for naive datetime objects with dates is close to the epoch. + # Thus, it is highly recommended to only use datetime *with* timezone information (no issue here). + # See bug reports: + # https://github.com/python/cpython/issues/81708 and https://github.com/python/cpython/issues/94414 + # The workaround is to go via timezone-aware datetime objects, which seem to work - with one caveat. + local_tz = datetime.now().astimezone().tzinfo + value = value.replace(tzinfo=local_tz) + value = value.astimezone(timezone.utc) + # Caveat: times may be off by; offset should be 0 but actually was seen at -3600 in CEST (Linux & Win). + # See also https://stackoverflow.com/q/56931738/551269 + # So, let's check value 0 as a reference and use the resulting timestamp as an offset for correction. + offset = datetime.fromtimestamp(0).replace(tzinfo=local_tz).astimezone(timezone.utc).timestamp() + return round((value.timestamp() - offset) * multiplier) # timestamp returns seconds + elif isinstance(value, float): + return round(value * multiplier) # floats typically represent seconds + elif isinstance(value, int): # Interpret ints as-is (without the multiplier); e.g. milliseconds or nanoseconds + return value + else: + raise TypeError( + f"Unsupported Python datetime type: {type(value)}. Please use datetime, float (seconds based) or " + f"int (milliseconds for Date, nanoseconds for DateNano).") diff --git a/objectbox/version.py b/objectbox/version.py index 79c63ad..ff95645 100644 --- a/objectbox/version.py +++ b/objectbox/version.py @@ -1,4 +1,4 @@ -# Copyright 2019-2021 ObjectBox Ltd. All rights reserved. +# Copyright 2019-2024 ObjectBox Ltd. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,16 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import * + class Version: - def __init__(self, major: int, minor: int, patch: int, label: str = ""): + def __init__(self, major: int, minor: int, patch: int, + alpha: Optional[int] = None, + beta: Optional[int] = None, + rc: Optional[int] = None): self.major = major self.minor = minor self.patch = patch - self.label = label + self.alpha = alpha + self.beta = beta + self.rc = rc def __str__(self): result = ".".join(map(str, [self.major, self.minor, self.patch])) - if len(self.label) > 0: - result += "-" + self.label + if self.alpha is not None: + result += f"a{self.alpha}" + if self.beta is not None: + result += f"b{self.beta}" + if self.rc is not None: + result += f"rc{self.rc}" return result diff --git a/requirements.txt b/requirements.txt index 38af119..7511ce5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ pip +setuptools wheel -flatbuffers==2.0 +flatbuffers==24.3.25 pytest>=4.4.1 -twine \ No newline at end of file +twine +numpy \ No newline at end of file diff --git a/setup.py b/setup.py index c052ffb..bda3122 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ python_requires='>=3.4, <4', license='Apache 2.0', classifiers=[ - "Development Status :: 3 - Alpha", + "Development Status :: 5 - Production/Stable" "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4", @@ -28,6 +28,9 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: C", "Programming Language :: C++", @@ -46,16 +49,19 @@ ], install_requires=[ - 'flatbuffers==2.0', + 'flatbuffers==24.3.25', + 'numpy' ], - packages=setuptools.find_packages(), + packages=setuptools.find_packages(exclude=['exampl*']), package_data={ 'objectbox': [ # Linux, macOS 'lib/x86_64/*', + 'lib/aarch64/*', 'lib/armv7l/*', 'lib/armv6l/*', + 'lib/macos-universal/*', # Windows 'lib/AMD64/*', ], diff --git a/tests/common.py b/tests/common.py index 2df4224..5a82e2e 100644 --- a/tests/common.py +++ b/tests/common.py @@ -1,41 +1,60 @@ -import objectbox import os -import shutil import pytest -from tests.model import TestEntity +import objectbox +from objectbox.logger import logger +from tests.model import * +import numpy as np +from datetime import datetime, timezone +from objectbox import * -test_dir = 'testdata' +def remove_json_model_file(): + path = os.path.dirname(os.path.realpath(__file__)) + json_file = os.path.join(path, "objectbox-model.json") + if os.path.exists(json_file): + os.remove(json_file) -def remove_test_dir(): - if os.path.exists(test_dir): - shutil.rmtree(test_dir) +def create_default_model(): + model = Model() + model.entity(TestEntity) + model.entity(TestEntityDatetime) + model.entity(TestEntityFlex) + model.entity(VectorEntity) + return model -# cleanup before and after each testcase -@pytest.fixture(autouse=True) -def autocleanup(): - remove_test_dir() - yield # run the test function - remove_test_dir() +def create_test_store(db_path: str = "testdata", clear_db: bool = True) -> objectbox.Store: + """ Creates a Store instance with all entities. """ + is_inmemory = db_path.startswith("memory:") + logger.info(f"DB path: {db_path} ({'in-memory' if is_inmemory else ''})") -def load_empty_test_objectbox(name: str = "") -> objectbox.ObjectBox: - model = objectbox.Model() - from objectbox.model import IdUid - model.entity(TestEntity, last_property_id=IdUid(10, 1010)) - model.last_entity_id = IdUid(1, 1) + if clear_db: + Store.remove_db_files(db_path) + remove_json_model_file() + return objectbox.Store(model=create_default_model(), directory=db_path) - db_name = test_dir if len(name) == 0 else test_dir + "/" + name - return objectbox.Builder().model(model).directory(db_name).build() +def assert_equal_prop(actual, expected, default): + if isinstance(expected, objectbox.model.Property): + assert (actual == default) + else: + assert (actual == expected) -def assert_equal_prop(actual, expected, default): - assert actual == expected or (isinstance( +def assert_equal_prop_vector(actual, expected, default): + assert (actual == np.array(expected)).all() or (isinstance( expected, objectbox.model.Property) and actual == default) +# compare approx values +def assert_equal_prop_approx(actual, expected, default): + if isinstance(expected, objectbox.model.Property): + assert (actual == default) + else: + assert (pytest.approx(actual) == expected) + + def assert_equal(actual: TestEntity, expected: TestEntity): """Check that two TestEntity objects have the same property data""" assert actual.id == expected.id @@ -46,3 +65,20 @@ def assert_equal(actual: TestEntity, expected: TestEntity): assert_equal_prop(actual.float64, expected.float64, 0) assert_equal_prop(actual.float32, expected.float32, 0) assert_equal_prop(actual.bytes, expected.bytes, b'') + assert_equal_prop_vector(actual.bools, expected.bools, np.array([])) + assert_equal_prop_vector(actual.ints, expected.ints, np.array([])) + assert_equal_prop_vector(actual.shorts, expected.shorts, np.array([])) + assert_equal_prop_vector(actual.chars, expected.chars, np.array([])) + assert_equal_prop_vector(actual.longs, expected.longs, np.array([])) + assert_equal_prop_vector(actual.floats, expected.floats, np.array([])) + assert_equal_prop_vector(actual.doubles, expected.doubles, np.array([])) + assert_equal_prop_approx(actual.bools_list, expected.bools_list, []) + assert_equal_prop_approx(actual.ints_list, expected.ints_list, []) + assert_equal_prop_approx(actual.shorts_list, expected.shorts_list, []) + assert_equal_prop_approx(actual.chars_list, expected.chars_list, []) + assert_equal_prop_approx(actual.longs_list, expected.longs_list, []) + assert_equal_prop_approx(actual.floats_list, expected.floats_list, []) + assert_equal_prop_approx(actual.doubles_list, expected.doubles_list, []) + assert_equal_prop_approx(actual.date, expected.date, datetime.fromtimestamp(0, timezone.utc)) + assert_equal_prop(actual.date_nano, expected.date_nano, 0) + assert_equal_prop(actual.flex, expected.flex, None) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7c4bca4 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,21 @@ +import pytest +from objectbox.logger import logger +from common import * + + +# Fixtures in this file are used by all files in the same directory: +# https://docs.pytest.org/en/7.1.x/reference/fixtures.html#conftest-py-sharing-fixtures-across-multiple-files + + +@pytest.fixture(autouse=True) +def cleanup_db(): + # Not needed: every test clears the DB on start, without deleting it on exit (not necessary) + # Also, here we have no information regarding the DB path being used (although usually is "testdata") + pass + + +@pytest.fixture +def test_store(): + store = create_test_store() + yield store + store.close() diff --git a/tests/model.py b/tests/model.py index 8b26a80..4388cd0 100644 --- a/tests/model.py +++ b/tests/model.py @@ -1,19 +1,81 @@ -from objectbox.model import * +from objectbox import * - -@Entity(id=1, uid=1) +@Entity() class TestEntity: - id = Id(id=1, uid=1001) - str = Property(str, id=2, uid=1002) - bool = Property(bool, id=3, uid=1003) - int64 = Property(int, type=PropertyType.long, id=4, uid=1004) - int32 = Property(int, type=PropertyType.int, id=5, uid=1005) - int16 = Property(int, type=PropertyType.short, id=6, uid=1006) - int8 = Property(int, type=PropertyType.byte, id=7, uid=1007) - float64 = Property(float, type=PropertyType.double, id=8, uid=1008) - float32 = Property(float, type=PropertyType.float, id=9, uid=1009) - bytes = Property(bytes, id=10, uid=1010) + id = Id() + # TODO Enable indexing dynamically, e.g. have a constructor to enable index(es). + # E.g. indexString=False (defaults to false). Same for bytes. + # TODO Test HASH and HASH64 indices (only supported for strings) + str = String(index=Index()) + bool = Bool() + int64 = Int64(index=Index()) + int32 = Int32() + int16 = Int16() + int8 = Int8() + float64 = Float64() + float32 = Float32() + bools = BoolVector() + bytes = Int8Vector() + shorts = Int16Vector() + chars = CharVector() + ints = Int32Vector() + longs = Int64Vector() + floats = Float32Vector() + doubles = Float64Vector() + bools_list = BoolList() + shorts_list = Int16List() + chars_list = CharList() + ints_list = Int32List() + longs_list = Int64List() + floats_list = Float32List() + doubles_list = Float64List() + date = Date() + date_nano = DateNano(int) + flex = Flex() + bytes = Bytes() transient = "" # not "Property" so it's not stored - def __init__(self, string: str = ""): - self.str = string + +@Entity() +class TestEntityDatetime: + id = Id() + date = Date(float) + date_nano = DateNano() + + +@Entity() +class TestEntityFlex: + id = Id() + flex = Flex() + + +@Entity() +class VectorEntity: + id = Id() + name = String() + vector_euclidean = Float32Vector( + index=HnswIndex( + dimensions=2, distance_type=VectorDistanceType.EUCLIDEAN + ), + ) + vector_cosine = Float32Vector( + index=HnswIndex( + dimensions=2, distance_type=VectorDistanceType.COSINE + ), + ) + vector_dot_product = Float32Vector( + index=HnswIndex( + dimensions=2, distance_type=VectorDistanceType.DOT_PRODUCT + ), + ) + # TODO: dot-product non-normalized + #vector_dot_product_non_normalized = Float32Vector( + # id=6, + # uid=4006, + # index=HnswIndex( + # id=6, + # uid=40004, + # dimensions=2, + # distance_type=VectorDistanceType.DOT_PRODUCT_NON_NORMALIZED, + # ), + #) diff --git a/tests/test_basics.py b/tests/test_basics.py index 4f4b21c..a6738c7 100644 --- a/tests/test_basics.py +++ b/tests/test_basics.py @@ -1,11 +1,34 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import objectbox -from tests.common import autocleanup, load_empty_test_objectbox, assert_equal +from tests.common import create_test_store def test_version(): + assert objectbox.version.major == 4 # update for major version changes + assert objectbox.version.minor >= 0 + + assert objectbox.version_core.major == 4 # update for major version changes + assert objectbox.version_core.minor >= 0 + info = objectbox.version_info() + print("\nVersion found:", info) assert len(info) > 10 + assert str(objectbox.version) in info + assert str(objectbox.version_core) in info def test_open(): - load_empty_test_objectbox() + store = create_test_store() + store.close() diff --git a/tests/test_box.py b/tests/test_box.py index b252426..c33fe39 100644 --- a/tests/test_box.py +++ b/tests/test_box.py @@ -1,12 +1,15 @@ import pytest import objectbox -from tests.model import TestEntity -from tests.common import autocleanup, load_empty_test_objectbox, assert_equal +from tests.model import TestEntity, TestEntityDatetime, TestEntityFlex +from tests.common import * +import numpy as np +from datetime import datetime, timezone +import time +from math import floor -def test_box_basics(): - ob = load_empty_test_objectbox() - box = objectbox.Box(ob, TestEntity) +def test_box_basics(test_store): + box = test_store.box(TestEntity) assert box.is_empty() assert box.count() == 0 @@ -29,50 +32,75 @@ def test_box_basics(): object.float64 = 4.2 object.float32 = 1.5 object.bytes = bytes([1, 1, 2, 3, 5]) + object.bools = np.array([True, False, True, True, False], dtype=np.bool_) + object.ints = np.array([1, 2, 3, 555, 120, 222], dtype=np.int32) + object.shorts = np.array([7, 8, 9, 12, 13, 22], dtype=np.int16) + object.chars = np.array([311, 426, 852, 927, 1025], dtype=np.uint16) + object.longs = np.array([4299185519, 155462547, 5019238156195], dtype=np.int64) + object.floats = np.array([0.1, 1.2, 2.3, 3.4, 4.5], dtype=np.float32) + object.doubles = np.array([99.99, 88.88, 77.77, 66.66, 55.595425], dtype=np.float64) + object.bools_list = [True, False, True, True, False] + object.ints_list = [91, 82, 73, 64, 55] + object.shorts_list = [8, 2, 7, 3, 6] + object.chars_list = [4, 5, 43, 75, 12] + object.longs_list = [4568, 8714, 1234, 5678, 9012240941] + object.floats_list = [0.11, 1.22, 2.33, 3.44, 4.5595] + object.doubles_list = [99.1999, 88.2888, 77.3777, 66.4666, 55.6597555] + object.date = time.time() # seconds since UNIX epoch (float) + object.date_nano = time.time_ns() # nanoseconds since UNIX epoch (int) + object.flex = dict(a=1, b=2, c=3) object.transient = "abcd" id = box.put(object) assert id == 5 assert id == object.id - # check the count assert not box.is_empty() assert box.count() == 2 # read + # wrap date so it can be compared (is read as datetime) + object.date = datetime.fromtimestamp(round(object.date * 1000) / 1000, tz=timezone.utc) read = box.get(object.id) assert_equal(read, object) assert read.transient != object.transient # != # update object.str = "bar" + object.date = floor(time.time_ns() / 1000000) # check that date can also be an int + object.date_nano = time.time() # check that date_nano can also be a float id = box.put(object) assert id == 5 # read again read = box.get(object.id) - assert_equal(read, object) + assert read.str == "bar" + assert floor(read.date.timestamp() * 1000) == object.date + assert read.date_nano == floor(object.date_nano * 1000000000) # remove - box.remove(object) - box.remove(1) + success = box.remove(object) + assert success + + # remove should return success + success = box.remove(1) + assert success + success = box.remove(1) + assert success is False # check they're gone assert box.count() == 0 - with pytest.raises(objectbox.NotFoundException): - box.get(object.id) - with pytest.raises(objectbox.NotFoundException): - box.get(1) + assert box.get(object.id) is None + assert box.get(1) is None -def test_box_bulk(): - ob = load_empty_test_objectbox() - box = objectbox.Box(ob, TestEntity) +def test_box_bulk(test_store): + box = test_store.box(TestEntity) - box.put(TestEntity("first")) + box.put(TestEntity(str="first")) - objects = [TestEntity("second"), TestEntity("third"), - TestEntity("fourth"), box.get(1)] + objects = [TestEntity(str="second"), TestEntity(str="third"), + TestEntity(str="fourth"), box.get(1)] box.put(objects) assert box.count() == 4 assert objects[0].id == 2 @@ -80,7 +108,8 @@ def test_box_bulk(): assert objects[2].id == 4 assert objects[3].id == 1 - assert_equal(box.get(objects[0].id), objects[0]) + read = box.get(objects[0].id) + assert_equal(read, objects[0]) assert_equal(box.get(objects[1].id), objects[1]) assert_equal(box.get(objects[2].id), objects[2]) assert_equal(box.get(objects[3].id), objects[3]) @@ -96,3 +125,158 @@ def test_box_bulk(): removed = box.remove_all() assert removed == 4 assert box.count() == 0 + + +def test_datetime(test_store): + box = test_store.box(TestEntityDatetime) + + assert box.is_empty() + assert box.count() == 0 + + # creat - deferred for now, as there is an issue with 0 timestamp on Windows + # object = TestEntityDatetime() + # id = box.put(object) + # assert id == 1 + # assert id == object.id + + # create with a given ID and some data + object = TestEntityDatetime() + object.id = 5 + object.date = datetime.utcnow() # milliseconds since UNIX epoch + object.date_nano = datetime.utcnow() # nanoseconds since UNIX epoch + + id = box.put(object) + assert id == 5 + assert id == object.id + # check the count + assert not box.is_empty() + assert box.count() == 1 + + # read + read = box.get(object.id) + assert type(read.date) == float + assert type(read.date_nano) == datetime + assert pytest.approx(read.date) == object.date.timestamp() + + # update + object.date = datetime.utcnow() + object.date_nano = datetime.utcnow() + id = box.put(object) + assert id == 5 + + # read again + read = box.get(object.id) + assert pytest.approx(read.date) == object.date.timestamp() + assert pytest.approx(read.date_nano.timestamp()) == object.date_nano.timestamp() + + # remove + success = box.remove(object) + assert success + + # check they're gone + assert box.count() == 0 + assert box.get(object.id) is None + assert box.get(1) is None + + +def test_datetime_special_values(test_store): + box = test_store.box(TestEntityDatetime) + assert box.is_empty() + + object = TestEntityDatetime() + object.date = 0 + object.date_nano = 0.0 + id = box.put(object) + assert object.id == id + + read = box.get(id) + assert isinstance(read.date, float) + assert read.date == 0.0 + assert isinstance(read.date_nano, datetime) + assert read.date_nano == datetime.fromtimestamp(0, timezone.utc) + + object.date = datetime.fromtimestamp(1.0, timezone.utc) + object.date_nano = datetime.fromtimestamp(1.0, timezone.utc) + id = box.put(object) + + read = box.get(id) + assert isinstance(read.date, float) + assert read.date == 1.0 + assert isinstance(read.date_nano, datetime) + assert read.date_nano == datetime.fromtimestamp(1.0, timezone.utc) + + +def test_flex(test_store): + def test_put_get(object: TestEntity, box: objectbox.Box, property): + object.flex = property + id = box.put(object) + assert id == object.id + read = box.get(object.id) + assert read.flex == object.flex + + box = test_store.box(TestEntity) + object = TestEntity() + + # Put an empty object + id = box.put(object) + assert id == object.id + + # Put a None type object + test_put_get(object, box, None) + + # Update to int + test_put_get(object, box, 1) + + # Update to float + test_put_get(object, box, 1.2) + + # Update to string + test_put_get(object, box, "foo") + + # Update to int list + test_put_get(object, box, [1, 2, 3]) + + # Update to float list + test_put_get(object, box, [1.1, 2.2, 3.3]) + + # Update to dict + test_put_get(object, box, {"a": 1, "b": 2}) + + # Update to bool + test_put_get(object, box, True) + + # Update to dict inside dict + test_put_get(object, box, {"a": 1, "b": {"c": 2}}) + + # Update to list inside dict + test_put_get(object, box, {"a": 1, "b": [1, 2, 3]}) + + +def test_flex_values(test_store): + box = test_store.box(TestEntityFlex) + + # Test empty object + obj_id = box.put(TestEntityFlex()) + read_obj = box.get(obj_id) + assert read_obj.flex is None + + # Test int + obj_id = box.put(TestEntityFlex(flex=23)) + read_obj = box.get(obj_id) + assert read_obj.flex == 23 + + # Test string + obj_id = box.put(TestEntityFlex(flex="hello")) + read_obj = box.get(obj_id) + assert read_obj.flex == "hello" + + # Test mixed list + obj_id = box.put(TestEntityFlex(flex=[4, 5, 1, "foo", 23, "bar"])) + read_obj = box.get(obj_id) + assert read_obj.flex == [4, 5, 1, "foo", 23, "bar"] + + # Test dictionary + dict_ = {"a": 1, "b": {"list": [1, 2, 3], "int": 5}} + obj_id = box.put(TestEntityFlex(flex=dict_)) + read_obj = box.get(obj_id) + assert read_obj.flex == dict_ diff --git a/tests/test_deprecated.py b/tests/test_deprecated.py new file mode 100644 index 0000000..158238d --- /dev/null +++ b/tests/test_deprecated.py @@ -0,0 +1,33 @@ +import tests.common +from objectbox import ObjectBox +from objectbox.c import * +from objectbox.model.idsync import sync_model +from objectbox.store_options import StoreOptions +from tests.common import * + +def test_deprecated_ObjectBox(): + Store.remove_db_files("testdata") + remove_json_model_file() + + model = tests.common.create_default_model() + sync_model(model) # It expects IDs to be already assigned + + options = StoreOptions() + options.model(model) + options.directory("testdata") + c_store = obx_store_open(options._c_handle) + with pytest.deprecated_call(): + ob = ObjectBox(c_store) + box = objectbox.Box(ob, TestEntity) + assert box.count() == 0 + ob.close() # TODO The store shall be closed even if the test fails + + +def test_deprecated_Builder(): + Store.remove_db_files("testdata") + remove_json_model_file() + + model = tests.common.create_default_model() + with pytest.deprecated_call(): + ob = objectbox.Builder().model(model).directory("testdata").build() + ob.close() diff --git a/tests/test_hnsw.py b/tests/test_hnsw.py new file mode 100644 index 0000000..fdb2220 --- /dev/null +++ b/tests/test_hnsw.py @@ -0,0 +1,206 @@ +import math +import numpy as np +import random +from common import * +from objectbox.query_builder import QueryBuilder +from typing import * + +def _find_expected_nn(points: np.ndarray, query: np.ndarray, n: int): + """ Given a set of points of shape (N, P) and a query of shape (P), finds the n points nearest to query. """ + + assert points.ndim == 2 and query.ndim == 1 + assert points.shape[1] == query.shape[0] + + d = np.linalg.norm(points - query, axis=1) # Euclidean distance + return np.argsort(d)[:n] + + +def _test_random_points( + num_points: int, + num_query_points: int, + seed: Optional[int] = None, + distance_type: VectorDistanceType = VectorDistanceType.EUCLIDEAN, + min_score: float = 0.5): + """ Generates random points in a 2d plane; checks the queried NN against the expected. """ + + vector_field_name = "vector_"+distance_type.name.lower() + + print(f"Test random points; Points: {num_points}, Query points: {num_query_points}, Seed: {seed}") + + k = 10 + + if seed is not None: + np.random.seed(seed) + + points = np.random.rand(num_points, 2).astype(np.float32) + + test_store = create_test_store() + + # Init and seed DB + box = test_store.box(VectorEntity) + + print(f"Seeding DB with {num_points} points...") + objects = [] + for i in range(points.shape[0]): + object_ = VectorEntity() + object_.name = f"point_{i}" + setattr(object_, vector_field_name, points[i]) + objects.append(object_) + box.put(*objects) + print(f"DB seeded with {box.count()} random points!") + + assert box.count() == num_points + + # Generate a random list of query points + query_points = np.random.rand(num_query_points, 2).astype(np.float32) + + # Iterate query points, and compare expected result with OBX result + print(f"Running {num_query_points} searches...") + for i in range(query_points.shape[0]): + query_point = query_points[i] + + # Find the ground truth (brute force) + expected_result = _find_expected_nn(points, query_point, k) + 1 # + 1 because OBX IDs start from 1 + assert len(expected_result) == k + + # Run ANN with OBX + qb = box.query() + qb.nearest_neighbors_f32(vector_field_name, query_point, k) + query = qb.build() + obx_result = [id_ for id_, score in query.find_ids_with_scores()] # Ignore score + assert len(obx_result) == k + + # We would like at least half of the expected results, to be returned by the search (in any order) + # Remember: it's an approximate search! + search_score = len(np.intersect1d(expected_result, obx_result)) / k + assert search_score >= min_score # TODO likely could be increased + + print(f"Done!") + + test_store.close() + + +def test_random_points(): + + min_score = 0.5 + distance_type = VectorDistanceType.EUCLIDEAN + _test_random_points(num_points=100, num_query_points=10, seed=10, distance_type=distance_type, min_score=min_score) + _test_random_points(num_points=100, num_query_points=10, seed=11, distance_type=distance_type, min_score=min_score) + _test_random_points(num_points=100, num_query_points=10, seed=12, distance_type=distance_type, min_score=min_score) + _test_random_points(num_points=100, num_query_points=10, seed=13, distance_type=distance_type, min_score=min_score) + _test_random_points(num_points=100, num_query_points=10, seed=14, distance_type=distance_type, min_score=min_score) + _test_random_points(num_points=100, num_query_points=10, seed=15, distance_type=distance_type, min_score=min_score) + +def _test_combined_nn_search(test_store: Store, distance_type: VectorDistanceType = VectorDistanceType.EUCLIDEAN): + box = test_store.box(VectorEntity) + + vector_field_name = "vector_"+distance_type.name.lower() + + values = [ + ("Power of red", [1, 1]), + ("Blueberry", [2, 2]), + ("Red", [3, 3]), + ("Blue sea", [4, 4]), + ("Lightblue", [5, 5]), + ("Red apple", [6, 6]), + ("Hundred", [7, 7]), + ("Tired", [8, 8]), + ("Power of blue", [9, 9]) + ] + for value in values: + entity = VectorEntity() + setattr(entity, "name", value[0]) + setattr(entity, vector_field_name, value[1]) + box.put(entity) + + assert box.count() == 9 + + # Test condition + NN search + qb = box.query() + qb.nearest_neighbors_f32(vector_field_name, [4.1, 4.2], 6) + qb.contains_string("name", "red", case_sensitive=False) + query = qb.build() + # 4, 5, 3, 6, 2, 7 + # Filtered: 3, 6, 7 + search_results = query.find_with_scores() + assert len(search_results) == 3 + assert search_results[0][0].name == "Red" + assert search_results[1][0].name == "Red apple" + assert search_results[2][0].name == "Hundred" + + # Test offset/limit on find_with_scores (result is ordered by score desc) + query.offset(1) + query.limit(1) + search_results = query.find_with_scores() + assert len(search_results) == 1 + assert search_results[0][0].name == "Red apple" + + # Regular condition + NN search + qb = box.query() + qb.nearest_neighbors_f32(vector_field_name, [9.2, 8.9], 7) + qb.starts_with_string("name", "Blue", case_sensitive=True) + query = qb.build() + + search_results = query.find_with_scores() + assert len(search_results) == 1 + assert search_results[0][0].name == "Blue sea" + + # Regular condition + NN search + qb = box.query() + qb.nearest_neighbors_f32(vector_field_name, [7.7, 7.7], 8) + qb.contains_string("name", "blue", case_sensitive=False) + query = qb.build() + # 8, 7, 9, 6, 5, 4, 3, 2 + # Filtered: 9, 5, 4, 2 + search_results = query.find_ids_with_scores() + assert len(search_results) == 4 + assert search_results[0][0] == 9 + assert search_results[1][0] == 5 + assert search_results[2][0] == 4 + assert search_results[3][0] == 2 + + search_results = query.find_ids_by_score() + assert len(search_results) == 4 + assert search_results[0] == 9 + assert search_results[1] == 5 + assert search_results[2] == 4 + assert search_results[3] == 2 + + search_results = query.find_ids_by_score_numpy() + assert search_results.size == 4 + assert search_results[0] == 9 + assert search_results[1] == 5 + assert search_results[2] == 4 + assert search_results[3] == 2 + + search_results = query.find_ids() + assert len(search_results) == 4 + assert search_results[0] == 2 + assert search_results[1] == 4 + assert search_results[2] == 5 + assert search_results[3] == 9 + + # Test offset/limit on find_ids (result is ordered by ID asc) + query.offset(1) + query.limit(2) + search_results = query.find_ids() + assert len(search_results) == 2 + assert search_results[0] == 4 + assert search_results[1] == 5 + + # Test empty result + query.offset(999) + assert len(query.find_ids()) == 0 + assert len(query.find_ids_with_scores()) == 0 + assert len(query.find_ids_by_score()) == 0 + numpy_result = query.find_ids_by_score_numpy() + assert numpy_result.size == 0 + assert str(numpy_result.dtype) == "uint64" + assert len(numpy_result) == 0 + + +def test_combined_nn_search(test_store): + """ Tests NN search combined with regular query conditions, offset and limit. """ + distance_type = VectorDistanceType.EUCLIDEAN + _test_combined_nn_search(test_store, distance_type) + # TODO: Cosine, DotProduct diverges see below diff --git a/tests/test_idsync.py b/tests/test_idsync.py new file mode 100644 index 0000000..9b5298a --- /dev/null +++ b/tests/test_idsync.py @@ -0,0 +1,651 @@ +import json +import pytest +import os +from numpy.testing import assert_approx_equal +from objectbox import * +from objectbox.model import * +from objectbox.model.entity import _Entity +from objectbox.model.idsync import sync_model +from objectbox.c import CoreException +from os import path + +from tests.common import remove_json_model_file + + +class _TestEnv: + """ + Test setup/tear-down of model json files, db store and utils. + Starts "fresh" on construction: deletes the model json file and the db files. + """ + def __init__(self): + self.model_path = 'test.json' + if path.exists(self.model_path): + os.remove(self.model_path) + self._model = None + self.db_path = 'testdb' + Store.remove_db_files(self.db_path) + self._store = None # Last created store + + def sync(self, model: Model) -> bool: + """ Returns True if changes were made and the model JSON was written. """ + self._model = model + return sync_model(self._model, self.model_path) + + def json(self): + return json.load(open(self.model_path)) + + def create_store(self): + assert self._model is not None, "Model must be set before creating store" + if self._store is not None: + self._store.close() + self._store = Store(model=self._model, directory=self.db_path) + return self._store + + def close(self): + if self._store is not None: + self._store.close() + + +def reset_ids(entity: _Entity): + entity._iduid = IdUid(0, 0) + entity._last_property_iduid = IdUid(0, 0) + for prop in entity._properties: + prop.iduid = IdUid(0, 0) + if prop.index: + prop.index.iduid = IdUid(0, 0) + +@pytest.fixture +def env(): + env_ = _TestEnv() + yield env_ + env_.close() + + +def test_empty_model(env): + """ Tests situations where the user attempts to sync an empty model. """ + + # JSON file didn't exist, user syncs an empty model -> no JSON file is generated + model = Model() + with pytest.raises(ValueError): + assert not env.sync(model) + assert not path.exists(env.model_path) + + # Init the JSON file with an entity + @Entity() + class MyEntity: + id = Id() + model = Model() + model.entity(MyEntity) + assert env.sync(model) # Model JSON written + + # JSON file exists, user tries to sync an empty model: must fail with JSON file untouched + model = Model() + with pytest.raises(ValueError): + env.sync(model) + + doc = env.json() + assert len(doc['entities']) == 1 + assert doc['entities'][0]['id'] == str(MyEntity._iduid) + + +def test_json(env): + @Entity() + class MyEntity: + id = Id() + my_string = String() + my_string_indexed = String(index=Index()) + + model = Model() + model.entity(MyEntity) + env.sync(model) + doc = env.json() + # debug: pprint(doc) + + json_e0 = doc['entities'][0] + e0_id = json_e0['id'] + assert e0_id == str(MyEntity._iduid) + assert e0_id.startswith("1:") + assert json_e0['name'] == "MyEntity" + + json_p0 = json_e0['properties'][0] + p0_id = json_p0['id'] + assert p0_id == str(MyEntity._get_property('id').iduid) + assert p0_id.startswith("1:") + assert json_p0['name'] == "id" + assert json_p0['flags'] == 1 + assert json_p0.get('indexId') is None + + json_p1 = json_e0['properties'][1] + assert json_p1['id'] == str(MyEntity._get_property('my_string').iduid) + assert json_p1['name'] == "my_string" + assert json_p1.get('flags') is None + assert json_p1.get('indexId') is None + + json_p2 = json_e0['properties'][2] + assert json_p2['id'] == str(MyEntity._get_property('my_string_indexed').iduid) + assert json_p2['name'] == "my_string_indexed" + assert json_p2['flags'] == 8 + assert json_p2['indexId'] == str(MyEntity._get_property('my_string_indexed').index.iduid) + assert json_e0['lastPropertyId'] == json_p2['id'] + + assert doc['lastEntityId'] == e0_id + assert doc['lastIndexId'] == json_p2['indexId'] + + +def test_basics(env): + @Entity() + class MyEntity: + id = Id() + name = String() + + model = Model() + model.entity(MyEntity) + env.sync(model) + assert MyEntity._id == 1 + assert MyEntity._uid != 0 + entity_ids = str(MyEntity._iduid) + + # create new database and populate with two objects + store = env.create_store() + entityBox = store.box(MyEntity) + entityBox.put(MyEntity(name="foo"),MyEntity(name="bar")) + assert entityBox.count() == 2 + del entityBox + + # recreate model using existing model json and open existing database + model = Model() + @Entity() + class MyEntity: + id = Id() + name = String() + model.entity(MyEntity) + assert str(model.entities[0]._iduid) == "0:0" + env.sync(model) + assert str(model.entities[0]._iduid) == entity_ids + + # open existing database + store = env.create_store() + entityBox = store.box(MyEntity) + assert entityBox.count() == 2 + +def test_entity_add(env): + @Entity() + class MyEntity1: + id = Id() + name = String() + model = Model() + model.entity(MyEntity1) + env.sync(model) + e0_iduid = IdUid(MyEntity1._id, MyEntity1._uid) + store = env.create_store() + box = store.box(MyEntity1) + box.put( MyEntity1(name="foo"), MyEntity1(name="bar")) + assert box.count() == 2 + + @Entity() + class MyEntity2: + id = Id() + name = String() + value = Int64() + model = Model() + reset_ids(MyEntity1) + model.entity(MyEntity1) + model.entity(MyEntity2) + assert str(model.entities[0]._iduid) == "0:0" + env.sync(model) + assert model.entities[0]._iduid == e0_iduid + store = env.create_store() + box1 = store.box(MyEntity1) + assert box1.count() == 2 + box2 = store.box(MyEntity2) + box2.put( MyEntity2(name="foo"), MyEntity2(name="bar")) + assert box2.count() == 2 + +def test_entity_remove(env): + @Entity() + class MyEntity1: + id = Id() + name = String() + @Entity() + class MyEntity2: + id = Id() + name = String() + value = Int64() + model = Model() + model.entity(MyEntity1) + model.entity(MyEntity2) + env.sync(model) + store = env.create_store() + box1 = store.box(MyEntity1) + box1.put( MyEntity1(name="foo"), MyEntity1(name="bar")) + box2 = store.box(MyEntity2) + box2.put( MyEntity2(name="foo"), MyEntity2(name="bar")) + assert box1.count() == 2 + assert box2.count() == 2 + + # Re-create a model without MyEntity2 + + model = Model() + reset_ids(MyEntity1) + model.entity(MyEntity1) + env.sync(model) + store = env.create_store() + box1 = store.box(MyEntity1) + assert box1.count() == 2 + + # MyEntity2 is gone and should raise CoreException + with pytest.raises(CoreException): + box2 = store.box(MyEntity2) + +def test_entity_rename(env): + model = Model() + @Entity() + class MyEntity: + id = Id() + name = String() + model.entity(MyEntity) + env.sync(model) + + # Save uid of entity for renaming purposes.. + uid = MyEntity._uid # iduid.uid + assert uid != 0 + # Debug: print("UID: "+ str(uid)) + + store = env.create_store() + box = store.box(MyEntity) + box.put(MyEntity(name="foo"),MyEntity(name="bar")) + assert box.count() == 2 + del box + + @Entity(uid=uid) + class MyRenamedEntity: + id = Id() + name = String() + + model = Model() + model.entity(MyRenamedEntity) + env.sync(model) + store = env.create_store() + box = store.box(MyRenamedEntity) + assert box.count() == 2 + + +def test_entity_rename_2(env): + # Init JSON file + @Entity(uid=365) + class Entity1: + id = Id() + + @Entity(uid=324) + class Entity2: + id = Id() + + @Entity(uid=890) + class Entity3: + id = Id() + + model = Model() + model.entity(Entity1) + model.entity(Entity2) + model.entity(Entity3) + assert env.sync(model) + assert model.last_entity_iduid == IdUid(3, 890) + + # Rename Entity2 -> Entity4 (same UID) + @Entity(uid=324) + class Entity4: + id = Id() + name = String() # Add one property also + + model = Model() + reset_ids(Entity1) + reset_ids(Entity3) + model.entity(Entity1) + model.entity(Entity3) + model.entity(Entity4) + assert env.sync(model) + assert Entity4._iduid == IdUid(2, 324) # Same ID/UID of Entity2 (renaming) + assert model.last_entity_iduid == IdUid(3, 890) + + +def test_prop_add(env): + + @Entity() + class MyEntity: + id = Id() + name = String() + model = Model() + model.entity(MyEntity) + env.sync(model) + store = env.create_store() + box = store.box(MyEntity) + box.put( MyEntity(name="foo"), MyEntity(name="bar")) + del box + + @Entity() + class MyEntity: + id = Id() + name = String() + value = Property(int, type=PropertyType.int) + + model = Model() + model.entity(MyEntity) + env.sync(model) + store = env.create_store() + box = store.box(MyEntity) + + assert box.count() == 2 + +def test_prop_remove(env): + + @Entity() + class MyEntity: + id = Id() + name = String() + value = Property(int, type=PropertyType.int) + + model = Model() + model.entity(MyEntity) + env.sync(model) + store = env.create_store() + box = store.box(MyEntity) + box.put( MyEntity(name="foo"), MyEntity(name="bar")) + del box + + @Entity() + class MyEntity: + id = Id() + name = String() + + model = Model() + model.entity(MyEntity) + env.sync(model) + store = env.create_store() + box = store.box(MyEntity) + assert box.count() == 2 + + +def test_prop_rename(env): + @Entity() + class EntityA: + id = Id() + name = String() + + model = Model() + model.entity(EntityA) + env.sync(model) + store = env.create_store() + box = store.box(EntityA) + box.put(EntityA(name="Luca")) + assert box.count() == 1 + assert box.get(1).name == "Luca" + assert not hasattr(box.get(1), "renamed_name") + + entity1_iduid = EntityA._iduid + name = EntityA._get_property("name") + name_iduid = name.iduid + print(f"Entity.name ID/UID: {name.iduid}") + + del box # Close store + + # *** Rename *** + + @Entity() + class EntityA: + id = Id() + renamed_name = Property(str, uid=name.uid) # Renamed property (same UID as "name") + + model = Model() + model.entity(EntityA) + env.sync(model) + store = env.create_store() + + # Check ID/UID(s) are preserved after renaming + entity2_iduid = EntityA._iduid + renamed_name = EntityA._get_property("renamed_name") + renamed_name_iduid = renamed_name.iduid + print(f"Entity.renamed_name ID/UID: {renamed_name_iduid}") + assert entity1_iduid == entity2_iduid + assert name_iduid == renamed_name_iduid + + # Check property value is preserved after renaming + box = store.box(EntityA) + assert box.count() == 1 + assert not hasattr(box.get(1), "name") + assert box.get(1).renamed_name == "Luca" + + +def test_model_json_updates(env): + """ Tests situations where the model JSON should be written/should not be written. """ + + def sync_entities(*entities: _Entity): + model = Model() + for entity in entities: + model.entity(entity) + return env.sync(model) + + # Init + @Entity() + class EntityA: + id = Id() + name = String() + assert sync_entities(EntityA) + + # Add entity + @Entity() + class EntityB: + id = Id() + name = String() + assert sync_entities(EntityB) + + entityb_uid = EntityB._uid + + # Rename entity + @Entity(uid=entityb_uid) + class EntityC: + id = Id() + name = String() + assert sync_entities(EntityC) + + # Noop + model = Model() + model.entity(EntityC) + assert not env.sync(model) + + # Add entity + @Entity() + class EntityD: + id = Id() + name = String() + age = Int8() + assert sync_entities(EntityC, EntityD) + + # Noop + assert sync_entities(EntityC, EntityD) is False + + # Replace entity + @Entity() + class EntityE: + id = Id() + assert sync_entities(EntityD, EntityE) + + # Noop + assert sync_entities(EntityD, EntityE) is False + + # Remove entity + assert sync_entities(EntityD) + + # Noop + assert sync_entities(EntityD) is False + + # Add property + @Entity() + class EntityD: + id = Id() + name = String() + age = Int8() + my_prop = String() + assert sync_entities(EntityD) + + my_prop_uid = EntityD._get_property("my_prop").uid + + # Rename property + @Entity() + class EntityD: + id = Id() + name = String() + age = Int8() + my_prop_renamed = String(uid=my_prop_uid) + assert sync_entities(EntityD) + + # Noop + assert sync_entities(EntityD) is False + + # Remove property + @Entity() + class EntityD: + id = Id() + name = String() + age = Int8() + assert sync_entities(EntityD) + + +def test_model_uid_already_assigned(env): + """ Tests an invalid situation where the user supplies a UID which is already present elsewhere in the JSON. """ + + @Entity() + class EntityA: + id = Id() + prop = Property(str) + + model = Model() + model.entity(EntityA) + env.sync(model) + + entitya_uid = EntityA._uid + + # Rename property, but use a UID which is already assigned + @Entity() + class EntityA: + id = Id() + renamed_prop = Property(str, uid=entitya_uid) + + model = Model() + model.entity(EntityA) + with pytest.raises(ValueError) as e: + env.sync(model) + assert f"User supplied UID {entitya_uid} is already assigned elsewhere" == str(e.value) + + +def test_models_named(env): + @Entity(model="modelA") + class EntityA: + id = Id + text_a = String + + @Entity(model="modelB") + class EntityB: + id = Id + int_b = Int64 + + @Entity(model="modelB") + class EntityB2: + id = Id() + float_b = Float64 + + Store.remove_db_files("test-db-model-a") + Store.remove_db_files("test-db-model-b") + remove_json_model_file() + store_a = Store(model="modelA", directory="test-db-model-a") + remove_json_model_file() + store_b = Store(model="modelB", directory="test-db-model-b") + + box_a = store_a.box(EntityA) + id = box_a.put(EntityA(text_a="ah")) + assert id != 0 + assert box_a.get(id).text_a == "ah" + + # TODO to make this work we Store/Box to check if the type is actually registered. + # This might require to store the (Python) model in the Store. + # with pytest.raises(ValueError): + # store_a.box(EntityB) + + with pytest.raises(CoreException): + store_a.box(EntityB2) + + box_b = store_b.box(EntityB) + id = box_b.put(EntityB(int_b=42)) + assert id != 0 + assert box_b.get(id).int_b == 42 + + box_b2 = store_b.box(EntityB2) + id = box_b2.put(EntityB2(float_b=3.141)) + assert id != 0 + assert_approx_equal(box_b2.get(id).float_b, 3.141) + + # TODO to make this work we Store/Box to check if the type is actually registered. + # This might require to store the (Python) model in the Store. + # with pytest.raises(ValueError): + # store_b.box(EntityA) +def test_sync_dynamic_entities(env): + def create_entity(entity_name: str, dimensions: int, distance_type: VectorDistanceType, uid=0): + DynamicEntity = type(entity_name, (), { + "id": Id(), + "name": String(), + "vector": Float32Vector(index=HnswIndex(dimensions=dimensions, distance_type=distance_type)) + }) + return Entity(uid=uid)(DynamicEntity) # Apply @Entity decorator + + CosineVectorEntity = create_entity("CosineVectorEntity", + dimensions=2, + distance_type=VectorDistanceType.COSINE) + EuclideanVectorEntity = create_entity("EuclideanVectorEntity", + dimensions=2, + distance_type=VectorDistanceType.EUCLIDEAN) + DotProductEntity = create_entity("DotProductEntity", + dimensions=2, + distance_type=VectorDistanceType.DOT_PRODUCT_NON_NORMALIZED) + model = Model() + model.entity(CosineVectorEntity) + model.entity(EuclideanVectorEntity) + model.entity(DotProductEntity) + assert env.sync(model) + CosineVectorEntity_iduid = CosineVectorEntity._iduid + + store = env.create_store() + cosine_box = store.box(CosineVectorEntity) + cosine_box.put(CosineVectorEntity(name="CosineObj1", vector=[2, 1])) + cosine_box.put(CosineVectorEntity(name="CosineObj2", vector=[-6, 0])) + euclidean_box = store.box(EuclideanVectorEntity) + euclidean_box.put(EuclideanVectorEntity(name="EuclideanObj1", vector=[5, 4])) + euclidean_box.put(EuclideanVectorEntity(name="EuclideanObj2", vector=[2, -6])) + dot_product_box = store.box(DotProductEntity) + dot_product_box.put(DotProductEntity(name="DotProductObj1", vector=[10, 0])) + assert cosine_box.get(1).name == "CosineObj1" + assert cosine_box.get(2).name == "CosineObj2" + assert euclidean_box.get(1).name == "EuclideanObj1" + assert euclidean_box.get(2).name == "EuclideanObj2" + assert dot_product_box.get(1).name == "DotProductObj1" + + del cosine_box + del euclidean_box + del dot_product_box + del store + + # Rename CosineVectorEntity to MyCosineVectorEntity + MyCosineVectorEntity = create_entity("MyCosineVectorEntity", + dimensions=2, + distance_type=VectorDistanceType.COSINE, + uid=CosineVectorEntity_iduid.uid) + model = Model() + model.entity(MyCosineVectorEntity) + model.entity(EuclideanVectorEntity) + model.entity(DotProductEntity) + assert env.sync(model) + assert CosineVectorEntity_iduid == MyCosineVectorEntity._iduid + + # Check MyCosineVectorEntity objects are preserved after renaming + store = env.create_store() + cosine_box = store.box(MyCosineVectorEntity) + assert cosine_box.get(1).name == "CosineObj1" + assert cosine_box.get(2).name == "CosineObj2" diff --git a/tests/test_index.py b/tests/test_index.py new file mode 100644 index 0000000..d0f1e45 --- /dev/null +++ b/tests/test_index.py @@ -0,0 +1,47 @@ +import objectbox +from objectbox.model import * +from objectbox.model.properties import IndexType +import pytest +from tests.model import TestEntity +from tests.common import * + + +# TODO tests disabled because Python indices API changed, now they actually interact with the C API +# Fix tests to verify indices are set in the C model, and not only Python's (i.e. query the C API)! + +@pytest.mark.skip(reason="Test indices implementation") +def test_index_basics(test_store): + box = test_store.box(TestEntity) + + # create + object = TestEntity() + box.put(object) + + # string - default index type is hash + assert box._entity._properties[1]._index_type == IndexType.hash + + # int64 - default index type is value + assert box._entity._properties[3]._index_type == IndexType.value + + # int32 - index type overwritten to hash + assert box._entity._properties[4]._index_type == IndexType.hash + + # int16 - specify index type w/o explicitly enabling index + assert box._entity._properties[5]._index_type == IndexType.hash + + # bytes - index type overwritten to hash64 + assert box._entity._properties[10]._index_type == IndexType.hash64 + + +@pytest.mark.skip(reason="Test indices implementation") +def test_index_error(): + @Entity() + class TestEntityInvalidIndex: + id = Id() + + # Cannot set index type when index is False + try: + str = Property(str, index=False, index_type=IndexType.hash) + except Exception: + assert pytest.raises(Exception, + match='trying to set index type on property of id 2 while index is set to False') diff --git a/tests/test_inmemory.py b/tests/test_inmemory.py new file mode 100644 index 0000000..9a26408 --- /dev/null +++ b/tests/test_inmemory.py @@ -0,0 +1,31 @@ +import objectbox +from tests.common import create_test_store +from tests.model import TestEntity +import os.path +import shutil + + +def test_inmemory(): + # Use default path for persistent store + db_name = "testdata" + store = create_test_store(db_name) + box = store.box(TestEntity) + object = TestEntity() + id = box.put(object) + assert id == 1 + assert id == object.id + assert os.path.exists(db_name) + del box + store.close() + shutil.rmtree(db_name) + + # Expect no path for in-memory store + db_name = "memory:testdata" + store = create_test_store(db_name) + box = store.box(TestEntity) + object = TestEntity() + id = box.put(object) + assert id == 1 + assert id == object.id + assert not os.path.exists(db_name) + store.close() diff --git a/tests/test_internals.py b/tests/test_internals.py new file mode 100644 index 0000000..23907fd --- /dev/null +++ b/tests/test_internals.py @@ -0,0 +1,106 @@ +from objectbox import * +from objectbox.model.idsync import sync_model + +import os +import os.path +import pytest + +class _TestEnv: + """ + Test setup/tear-down of model json files, db store and utils. + Starts "fresh" on construction: deletes the model json file and the db files. + """ + def __init__(self): + self.model_path = 'test.json' + if os.path.exists(self.model_path): + os.remove(self.model_path) + self.model = None + self.db_path = 'testdb' + Store.remove_db_files(self.db_path) + + def sync(self, model: Model) -> bool: + """ Returns True if changes were made and the model JSON was written. """ + self.model = model + return sync_model(self.model, self.model_path) + + def store(self): + assert self.model is not None + return Store(model=self.model, directory=self.db_path) + +@pytest.fixture +def env(): + return _TestEnv() + + +def test_property_name_clash(env): + @Entity() + class MyEntity: + id = Id() + user_type = String() + iduid = String() + name = String() + last_property_id = String() + properties = String() + offset_properties = String() + id_property = String() + _id = String() # a bad one; this one don't work directly + a_safe_one = String() + + model = Model() + model.entity(MyEntity) + env.sync(model) + store = env.store() + + box = store.box(MyEntity) + id1 = box.put( + MyEntity( + user_type="foobar", + iduid="123", + name="bar", + last_property_id="blub", + properties="baz", + offset_properties="blah", + id_property = "kazong", + _id="fooz", + a_safe_one="blah", + ) + ) + assert box.count() == 1 + + assert len(box.query(MyEntity.id.equals(id1)).build().find()) == 1 + assert len(box.query(MyEntity.iduid.equals("123")).build().find()) == 1 + assert len(box.query(MyEntity.user_type.equals("foobar")).build().find()) == 1 + assert len(box.query(MyEntity.name.equals("bar")).build().find()) == 1 + assert len(box.query(MyEntity.last_property_id.equals("blub")).build().find()) == 1 + assert len(box.query(MyEntity.properties.equals("baz")).build().find()) == 1 + assert len(box.query(MyEntity.offset_properties.equals("blah")).build().find()) == 1 + assert len(box.query(MyEntity.id_property.equals("kazong")).build().find()) == 1 + with pytest.raises(AttributeError): + MyEntity._id.equals("fooz") + assert len(box.query(MyEntity._get_property("_id").equals("fooz")).build().find()) == 1 + assert len(box.query(MyEntity.a_safe_one.equals("blah")).build().find()) == 1 + + +def test_entity_attribute_methods_nameclash_check(): + # Test ensures we do not leave occasional instance attributes or class methods/attributes in + # helper class _Entity which might collide with user-defined property names. + # (We expect users not use use underscore to guarantee convient access to properties as-is via '.' operator) + + # To check instance as well as class data, we create a dummy entity which we'll scan next. + @Entity() + class MyEntity: + id = Id() + + not_prefixed = [] + + for attrname in MyEntity.__dict__: + if not attrname.startswith("_"): + not_prefixed.append(attrname) + + for methodname in MyEntity.__class__.__dict__: + if not methodname.startswith("_"): + not_prefixed.append(methodname) + + assert ( + len(not_prefixed) == 0 + ), f"INTERNAL: Public attributes/methods(s) detected in Class _Entity: {not_prefixed}\nPlease prefix with '_' to prevent name-collision with Property field-names." diff --git a/tests/test_model.py b/tests/test_model.py new file mode 100644 index 0000000..34a9f35 --- /dev/null +++ b/tests/test_model.py @@ -0,0 +1,83 @@ +from objectbox.model import * +from objectbox import * +from objectbox.model.idsync import sync_model +import os +import os.path + +def test_reuse_model(): + @Entity() + class MyEntity: + id = Id() + name = String() + + model = Model() + model.entity(MyEntity) + model_filepath = "test-model.json" + if os.path.exists(model_filepath): + os.remove(model_filepath) + sync_model(model, model_filepath) + + db1path = "test-db1" + db2path = "test-db2" + Store.remove_db_files(db1path) + Store.remove_db_files(db2path) + + store1 = Store(model=model, directory=db1path) + store2 = Store(model=model, directory=db2path) + + store1.close() + store2.close() + + +def test_reuse_entity(): + @Entity() + class MyEntity: + id = Id() + name = String() + + m1 = Model() + m1.entity(MyEntity) + model_filepath = "test-model1.json" + if os.path.exists(model_filepath): + os.remove(model_filepath) + sync_model(m1, model_filepath) + + db1path = "test-db1" + db2path = "test-db2" + Store.remove_db_files(db1path) + Store.remove_db_files(db2path) + + store1 = Store(model=m1, directory=db1path) + + box1 = store1.box(MyEntity) + box1.put(MyEntity(name="foo")) + assert box1.count() == 1 + + m2 = Model() + + @Entity() + class MyEntity2: + id = Id() + name = String() + value = Int64() + + m2.entity(MyEntity2) + m2.entity(MyEntity) + model_filepath = "test-model2.json" + if os.path.exists(model_filepath): + os.remove(model_filepath) + sync_model(m2, model_filepath) + + store2 = Store(model=m2, directory=db2path) + box2 = store2.box(MyEntity) + box2.put(MyEntity(name="bar")) + box2.put(MyEntity(name="bar")) + box2.put(MyEntity(name="bar")) + assert box2.count() == 3 + + box1.put(MyEntity(name="foo")) + box1.put(MyEntity(name="foo")) + assert box1.count() == 3 + + store1.close() + store2.close() diff --git a/tests/test_query.py b/tests/test_query.py new file mode 100644 index 0000000..706b800 --- /dev/null +++ b/tests/test_query.py @@ -0,0 +1,617 @@ +import objectbox +from objectbox import * +from objectbox.c import * +from objectbox.query import * +import pytest +from tests.common import * +from tests.model import * + + +def test_basics(test_store): + box_test_entity = test_store.box(TestEntity) + id1 = box_test_entity.put(TestEntity(bool=True, str="foo", int64=123)) + id2 = box_test_entity.put(TestEntity(bool=False, str="bar", int64=456)) + + box_vector_entity = test_store.box(VectorEntity) + box_vector_entity.put(VectorEntity(name="Object 1", vector_euclidean=[1, 1])) + box_vector_entity.put(VectorEntity(name="Object 2", vector_euclidean=[2, 2])) + box_vector_entity.put(VectorEntity(name="Object 3", vector_euclidean=[3, 3])) + + # Id query + query = box_test_entity.query(TestEntity.id.equals(id1)).build() + assert query.count() == 1 + query = box_test_entity.query(TestEntity.id.greater_than(id1)).build() + assert query.count() == 1 + query = box_test_entity.query(TestEntity.id.greater_or_equal(id1)).build() + assert query.count() == 2 + query = box_test_entity.query(TestEntity.id.less_than(id2)).build() + assert query.count() == 1 + query = box_test_entity.query(TestEntity.id.less_or_equal(id2)).build() + assert query.count() == 2 + + # Bool query + query = box_test_entity.query(TestEntity.bool.equals(True)).build() + assert query.count() == 1 + assert query.find()[0].id == id1 + + query = box_test_entity.query(TestEntity.bool.equals(False)).build() + assert query.count() == 1 + assert query.find()[0].id == id2 + + # String query + str_prop: Property = TestEntity.str + + # Case Sensitive = True + query = box_test_entity.query(str_prop.equals("bar", case_sensitive=True)).build() + # String query using direct `.` notation + + query = box_test_entity.query(TestEntity.str.equals("bar", case_sensitive=True)).build() + assert query.count() == 1 + assert query.find()[0].str == "bar" + + query = box_test_entity.query(TestEntity.str.not_equals("bar", case_sensitive=True)).build() + assert query.count() == 1 + assert query.find()[0].str == "foo" + + query = box_test_entity.query(TestEntity.str.contains("ba", case_sensitive=True)).build() + assert query.count() == 1 + assert query.find()[0].str == "bar" + + query = box_test_entity.query(TestEntity.str.starts_with("f", case_sensitive=True)).build() + assert query.count() == 1 + assert query.find()[0].str == "foo" + + query = box_test_entity.query(TestEntity.str.ends_with("o", case_sensitive=True)).build() + assert query.count() == 1 + assert query.find()[0].str == "foo" + + query = box_test_entity.query(TestEntity.str.greater_than("bar", case_sensitive=True)).build() + assert query.count() == 1 + assert query.find()[0].str == "foo" + + query = box_test_entity.query(TestEntity.str.greater_or_equal("bar", case_sensitive=True)).build() + assert query.count() == 2 + assert query.find()[0].str == "foo" + assert query.find()[1].str == "bar" + + query = box_test_entity.query(TestEntity.str.less_than("foo", case_sensitive=True)).build() + assert query.count() == 1 + assert query.find()[0].str == "bar" + + query = box_test_entity.query(TestEntity.str.less_or_equal("foo", case_sensitive=True)).build() + assert query.count() == 2 + assert query.find()[0].str == "foo" + assert query.find()[1].str == "bar" + + # Case Sensitive = False + + query = box_test_entity.query(TestEntity.str.equals("Bar", case_sensitive=False)).build() + assert query.count() == 1 + assert query.find()[0].str == "bar" + + query = box_test_entity.query(TestEntity.str.not_equals("Bar", case_sensitive=False)).build() + assert query.count() == 1 + assert query.find()[0].str == "foo" + + query = box_test_entity.query(TestEntity.str.contains("Ba", case_sensitive=False)).build() + assert query.count() == 1 + assert query.find()[0].str == "bar" + + query = box_test_entity.query(TestEntity.str.starts_with("F", case_sensitive=False)).build() + assert query.count() == 1 + assert query.find()[0].str == "foo" + + query = box_test_entity.query(TestEntity.str.ends_with("O", case_sensitive=False)).build() + assert query.count() == 1 + assert query.find()[0].str == "foo" + + query = box_test_entity.query(TestEntity.str.greater_than("BAR", case_sensitive=False)).build() + assert query.count() == 1 + assert query.find()[0].str == "foo" + + query = box_test_entity.query(TestEntity.str.greater_or_equal("BAR", case_sensitive=False)).build() + assert query.count() == 2 + assert query.find()[0].str == "foo" + assert query.find()[1].str == "bar" + + query = box_test_entity.query(TestEntity.str.less_than("FOo", case_sensitive=False)).build() + assert query.count() == 1 + assert query.find()[0].str == "bar" + + query = box_test_entity.query(TestEntity.str.less_or_equal("FoO", case_sensitive=False)).build() + assert query.count() == 2 + assert query.find()[0].str == "foo" + assert query.find()[1].str == "bar" + + # Int queries using a reference to property + int_prop: Property = TestEntity.int64 + + query = box_test_entity.query(int_prop.equals(123)).build() + assert query.count() == 1 + assert query.find()[0].int64 == 123 + + query = box_test_entity.query(int_prop.not_equals(123)).build() + assert query.count() == 1 + assert query.find()[0].int64 == 456 + + query = box_test_entity.query(int_prop.greater_than(123)).build() + assert query.count() == 1 + assert query.find()[0].int64 == 456 + + query = box_test_entity.query(int_prop.greater_or_equal(123)).build() + assert query.count() == 2 + assert query.find()[0].int64 == 123 + assert query.find()[1].int64 == 456 + + query = box_test_entity.query(int_prop.less_than(456)).build() + assert query.count() == 1 + assert query.find()[0].int64 == 123 + + query = box_test_entity.query(int_prop.less_or_equal(456)).build() + assert query.count() == 2 + assert query.find()[0].int64 == 123 + assert query.find()[1].int64 == 456 + + query = box_test_entity.query(int_prop.between(100, 200)).build() + assert query.count() == 1 + assert query.find()[0].int64 == 123 + + # + assert query.remove() == 1 + + # NN query + query = box_vector_entity.query(VectorEntity.vector_euclidean.nearest_neighbor([2.1, 2.1], 2)).build() + assert query.count() == 2 + assert query.find_ids() == [2, 3] + +def test_integer_scalars(test_store): + box_test_entity = test_store.box(TestEntity) + id1 = box_test_entity.put(TestEntity(int8=12, int16=12, int32=12, int64=12)) + id2 = box_test_entity.put(TestEntity(int8=45, int16=45, int32=45, int64=45)) + + props = [ "int8", "int16", "int32", "int64"] + for p in props: + prop = TestEntity._get_property(p) + + query = box_test_entity.query(prop.equals(12)).build() + assert query.count() == 1 + assert query.find()[0].id == id1 + + query = box_test_entity.query(prop.equals(45)).build() + assert query.count() == 1 + assert query.find()[0].id == id2 + + query = box_test_entity.query(prop.not_equals(12)).build() + assert query.count() == 1 + assert query.find()[0].id == id2 + + query = box_test_entity.query(prop.greater_than(12)).build() + assert query.count() == 1 + assert query.find()[0].id == id2 + + query = box_test_entity.query(prop.greater_or_equal(12)).build() + assert query.count() == 2 + assert query.find()[0].id == id1 + assert query.find()[1].id == id2 + + query = box_test_entity.query(prop.less_than(45)).build() + assert query.count() == 1 + assert query.find()[0].id == id1 + + query = box_test_entity.query(prop.less_or_equal(45)).build() + assert query.count() == 2 + assert query.find()[0].id == id1 + assert query.find()[1].id == id2 + +def test_float_scalars(test_store): + box_test_entity = test_store.box(TestEntity) + id1 = box_test_entity.put(TestEntity(float32=12.3, float64=12.3)) + id2 = box_test_entity.put(TestEntity(float32=45.6, float64=45.6)) + id3 = box_test_entity.put(TestEntity(float32=45.7, float64=45.7)) + + # Test int scalar literals + props = [ "float32", "float64" ] + for p in props: + prop = TestEntity._get_property(p) + + # equals/not_equals should not exist + with pytest.raises(AttributeError): + prop.equals(12) + with pytest.raises(AttributeError): + prop.not_equals(12) + + query = box_test_entity.query(prop.greater_or_equal(12)).build() + assert query.count() == 3 + query = box_test_entity.query(prop.greater_than(13)).build() + assert query.count() == 2 + assert query.find()[0].id == id2 + query = box_test_entity.query(prop.less_than(46)).build() + assert query.count() == 3 + query = box_test_entity.query(prop.less_or_equal(45)).build() + assert query.count() == 1 + query = box_test_entity.query(prop.between(10, 50)).build() + assert query.count() == 3 + query = box_test_entity.query(prop.between(12, 13)).build() + assert query.count() == 1 + query = box_test_entity.query(prop.between(12, 12)).build() + assert query.count() == 0 + + # Test float scalar literals + for p in props: + prop = TestEntity._get_property(p) + query = box_test_entity.query(prop.less_or_equal(12.299999)).build() + assert query.count() == 0 + query = box_test_entity.query(prop.greater_than(12.3)).build() + assert query.count() == 2 + query = box_test_entity.query(prop.greater_or_equal(12.3)).build() + assert query.count() == 3 + query = box_test_entity.query(prop.less_than(45.6)).build() + assert query.count() == 1 + query = box_test_entity.query(prop.less_or_equal(45.6)).build() + assert query.count() == 2 + query = box_test_entity.query(prop.between(12.2, 12.4)).build() + assert query.count() == 1 + query = box_test_entity.query(prop.between(45.5999, 45.61)).build() + assert query.count() == 1 + query = box_test_entity.query(prop.between(45.5999, 45.7001)).build() + assert query.count() == 2 + + +def test_flex_contains_key_value(test_store): + box = test_store.box(TestEntityFlex) + box.put(TestEntityFlex(flex={"k1": "String", "k2": 2, "k3": "string"})) + box.put(TestEntityFlex(flex={"k1": "strinG", "k2": 3, "k3": 10, "k4": [1, "foo", 3]})) + box.put(TestEntityFlex(flex={"k1": "buzz", "k2": 3, "k3": [2, 3], "k4": {"k1": "a", "k2": "inner text"}})) + box.put(TestEntityFlex(flex={"n1": "string", "n2": -7, "n3": [-10, 10], "n4": [4, 4, 4]})) + box.put(TestEntityFlex(flex={"n1": "Apple", "n2": 3, "n3": [2, 3, 5], "n4": {"n1": [1, 2, "bar"]}})) + + assert box.count() == 5 + + # Search case-sensitive = False + query = box.query(TestEntityFlex.flex.contains_key_value("k1", "string", False)).build() + results = query.find() + assert len(results) == 2 + assert results[0].flex["k1"] == "String" + assert results[0].flex["k2"] == 2 + assert results[0].flex["k3"] == "string" + assert results[1].flex["k1"] == "strinG" + assert results[1].flex["k2"] == 3 + assert results[1].flex["k3"] == 10 + assert results[1].flex["k4"] == [1, "foo", 3] + + # Search case-sensitive = True + query = box.query(TestEntityFlex.flex.contains_key_value("n1", "string", True)).build() + results = query.find() + assert len(results) == 1 + assert results[0].flex["n1"] == "string" + assert results[0].flex["n2"] == -7 + assert results[0].flex["n3"] == [-10, 10] + assert results[0].flex["n4"] == [4, 4, 4] + + # TODO Search using nested key (not supported yet) + + # No match (key) + query = box.query(TestEntityFlex.flex.contains_key_value("missing key", "string", True)).build() + assert len(query.find()) == 0 + + # No match (value) + query = box.query(TestEntityFlex.flex.contains_key_value("k1", "missing value", True)).build() + assert len(query.find()) == 0 + + +def test_offset_limit(test_store): + box = test_store.box(TestEntity) + box.put(TestEntity()) + box.put(TestEntity(str="a")) + box.put(TestEntity(str="b")) + box.put(TestEntity(str="c")) + assert box.count() == 4 + + int_prop = TestEntity.int64 + + query = box.query(int_prop.equals(0)).build() + assert query.count() == 4 + + query.offset(2) + assert len(query.find()) == 2 + assert query.find()[0].str == "b" + assert query.find()[1].str == "c" + + query.limit(1) + assert len(query.find()) == 1 + assert query.find()[0].str == "b" + + query.offset(0) + query.limit(0) + assert len(query.find()) == 4 + + +def test_any_all(test_store): + box = test_store.box(TestEntity) + + box.put(TestEntity(str="Foo", int32=10, int8=2, float32=3.14, bool=True)) + box.put(TestEntity(str="FooBar", int32=100, int8=50, float32=2.0, bool=True)) + box.put(TestEntity(str="Bar", int32=99, int8=127, float32=1.0, bool=False)) + box.put(TestEntity(str="Test", int32=1, int8=1, float32=0.0001, bool=True)) + box.put(TestEntity(str="test", int32=3232, int8=88, float32=1.0101, bool=False)) + box.put(TestEntity(str="Foo or BAR?", int32=0, int8=0, float32=0.0, bool=False)) + box.put(TestEntity(str="Just a test", int32=6, int8=6, float32=6.111, bool=False)) + box.put(TestEntity(str="EXAMPLE", int32=37, int8=37, float32=100, bool=True)) + + # Test all + qb = box.query() + qb.all([ + qb.starts_with_string("str", "Foo"), + qb.equals_int("int32", 10) + ]) + query = qb.build() + ids = query.find_ids() + assert ids == [1] + + # Test any + qb = box.query() + qb.any([ + qb.starts_with_string("str", "Test", case_sensitive=False), + qb.ends_with_string("str", "?"), + qb.equals_int("int32", 37) + ]) + query = qb.build() + ids = query.find_ids() + # 4, 5, 6, 8 + assert ids == [4, 5, 6, 8] + + # Test all/any + qb = box.query() + qb.any([ + qb.all([qb.contains_string("str", "Foo"), qb.less_than_int("int32", 100)]), + qb.equals_string("str", "Test", case_sensitive=False) + ]) + query = qb.build() + ids = query.find_ids() + # 1, 4, 5, 6 + assert ids == [1, 4, 5, 6] + + # Test all/any + qb = box.query() + qb.all([ + qb.any([ + qb.contains_string("str", "foo", case_sensitive=False), + qb.contains_string("str", "bar", case_sensitive=False) + ]), + qb.greater_than_int("int8", 30) + ]) + query = qb.build() + ids = query.find_ids() + # 2, 3 + assert ids == [2, 3] + + +def test_set_parameter(test_store): + box_test_entity = test_store.box(TestEntity) + box_test_entity.put(TestEntity(str="Foo", int64=2, int32=703, int8=101)) + box_test_entity.put(TestEntity(str="FooBar", int64=10, int32=49, int8=45)) + box_test_entity.put(TestEntity(str="Bar", int64=10, int32=226, int8=126)) + box_test_entity.put(TestEntity(str="Foster", int64=2, int32=301, int8=42)) + box_test_entity.put(TestEntity(str="Fox", int64=10, int32=157, int8=11)) + box_test_entity.put(TestEntity(str="Barrakuda", int64=4, int32=386, int8=60)) + + box_vector_entity = test_store.box(VectorEntity) + box_vector_entity.put(VectorEntity(name="Object 1", vector_euclidean=[1, 1])) + box_vector_entity.put(VectorEntity(name="Object 2", vector_euclidean=[2, 2])) + box_vector_entity.put(VectorEntity(name="Object 3", vector_euclidean=[3, 3])) + box_vector_entity.put(VectorEntity(name="Object 4", vector_euclidean=[4, 4])) + box_vector_entity.put(VectorEntity(name="Object 5", vector_euclidean=[5, 5])) + + qb = box_test_entity.query() + qb.starts_with_string("str", "fo", case_sensitive=False) + qb.greater_than_int("int32", 150) + qb.greater_than_int("int64", 0) + query = qb.build() + assert query.find_ids() == [1, 4, 5] + + # Test set_parameter_string + query.set_parameter_string("str", "bar") + assert query.find_ids() == [3, 6] + + # Test set_parameter_int + query.set_parameter_int("int64", 8) + assert query.find_ids() == [3] + + qb = box_vector_entity.query() + qb.nearest_neighbors_f32("vector_euclidean", [3.4, 3.4], 3) + query = qb.build() + assert query.find_ids() == sorted([3, 4, 2]) + + # set_parameter_vector_f32 + # set_parameter_int (NN count) + query.set_parameter_vector_f32("vector_euclidean", [4.9, 4.9]) + assert query.find_ids() == sorted([5, 4, 3]) + + query.set_parameter_vector_f32("vector_euclidean", [0, 0]) + assert query.find_ids() == sorted([1, 2, 3]) + + query.set_parameter_vector_f32("vector_euclidean", [2.5, 2.1]) + assert query.find_ids() == sorted([2, 3, 1]) + + query.set_parameter_int("vector_euclidean", 2) + assert query.find_ids() == sorted([2, 3]) + + +def test_set_parameter_alias(test_store): + box = test_store.box(TestEntity) + + box.put(TestEntity(str="Foo", int64=2, int32=703, int8=101)) + box.put(TestEntity(str="FooBar", int64=10, int32=49, int8=45)) + + box_vector = test_store.box(VectorEntity) + box_vector.put(VectorEntity(name="Object 1", vector_euclidean=[1, 1])) + box_vector.put(VectorEntity(name="Object 2", vector_euclidean=[2, 2])) + box_vector.put(VectorEntity(name="Object 3", vector_euclidean=[3, 3])) + box_vector.put(VectorEntity(name="Object 4", vector_euclidean=[4, 4])) + box_vector.put(VectorEntity(name="Object 5", vector_euclidean=[5, 5])) + + str_prop: Property = TestEntity.str + int32_prop: Property = TestEntity.int32 + int64_prop: Property = TestEntity.int64 + + # Test set parameter alias on string + qb = box.query(str_prop.equals("Foo").alias("foo_filter")) + query = qb.build() + + assert query.find()[0].str == "Foo" + assert query.count() == 1 + + query.set_parameter_alias_string("foo_filter", "FooBar") + assert query.find()[0].str == "FooBar" + assert query.count() == 1 + + # Test set parameter alias on int64 + qb = box.query(int64_prop.greater_than(5).alias("greater_than_filter")) + + query = qb.build() + assert query.count() == 1 + assert query.find()[0].str == "FooBar" + + query.set_parameter_alias_int("greater_than_filter", 1) + + assert query.count() == 2 + + # Test set parameter alias on string/int32 + query = box.query( + str_prop.equals("Foo").alias("str condition") + .and_(int32_prop.greater_than(700).alias("int32 condition")) + ).build() + + assert query.count() == 1 + assert query.find()[0].str == "Foo" + + query.set_parameter_alias_string("str condition", "FooBar") # FooBar int32 isn't higher than 700 (49) + assert query.count() == 0 + + query.set_parameter_alias_int("int32 condition", 40) + assert query.find()[0].str == "FooBar" + + # Test with & + query = box.query( + str_prop.equals("Foo").alias("str condition") + & int32_prop.greater_than(700).alias("int32 condition") + ).build() + assert query.count() == 1 + + # Test set parameter alias on vector + vector_prop: Property = VectorEntity.vector_euclidean + + query = box_vector.query(vector_prop.nearest_neighbor([3.4, 3.4], 3).alias("nearest_neighbour_filter")).build() + assert query.count() == 3 + assert query.find_ids() == sorted([3, 4, 2]) + + query.set_parameter_alias_vector_f32("nearest_neighbour_filter", [4.9, 4.9]) + assert query.count() == 3 + assert query.find_ids() == sorted([5, 4, 3]) + + +def test_set_parameter_alias_advanced(test_store): + """ Tests set_parameter_alias in a complex scenario (i.e. multiple query conditions/logical aggregations). """ + + # Setup 1 + box = test_store.box(TestEntity) + box.put(TestEntity(str="Apple", bool=False, int64=47, int32=70)) + box.put(TestEntity(str="applE", bool=True, int64=253, int32=798)) + box.put(TestEntity(str="APPLE", bool=False, int64=3456, int32=123)) + box.put(TestEntity(str="Orange", bool=False, int64=2345, int32=53)) + box.put(TestEntity(str="orange", bool=True, int64=546, int32=5678)) + box.put(TestEntity(str="ORANGE", bool=True, int64=78, int32=798)) + box.put(TestEntity(str="oRANGE", bool=True, int64=89, int32=1234)) + box.put(TestEntity(str="Zucchini", bool=False, int64=1234, int32=9)) + assert box.count() == 8 + + str_prop = TestEntity.str + bool_prop = TestEntity.bool + int32_prop = TestEntity.int32 + int64_prop = TestEntity.int64 + + query = box.query( + str_prop.equals("Dummy", case_sensitive=False).alias("str_filter") + .and_(bool_prop.equals(False).alias("bool_filter")) + .and_( + int64_prop.greater_than(0).alias("int64_filter") + .or_(int32_prop.less_than(100).alias("int32_filter")) + ) + ).build() + assert len(query.find_ids()) == 0 + + # Test & and | without alias + query = box.query( + str_prop.equals("applE") + | str_prop.equals("orange", case_sensitive=False) & bool_prop.equals(False) + ).build() + assert len(query.find_ids()) == 2 + + # Test using & and | ops + query = box.query( + str_prop.equals("Dummy", case_sensitive=False).alias("str_filter") + & bool_prop.equals(False).alias("bool_filter") + & ( + int64_prop.greater_than(0).alias("int64_filter") + | int32_prop.less_than(100).alias("int32_filter") + ) + ).build() + assert len(query.find_ids()) == 0 + + # TODO currently we don't support set_parameter_* for int32/bool/other types... + + query.set_parameter_alias_string("str_filter", "Apple") + query.set_parameter_alias_int("int64_filter", 300) + assert len(query.find_ids()) == 2 # Apple, APPLE + + query.set_parameter_alias_string("str_filter", "orange") + query.set_parameter_alias_int("int64_filter", 1000) + assert len(query.find_ids()) == 1 # Orange + + query.set_parameter_alias_string("str_filter", "Zucchini") + assert len(query.find_ids()) == 1 # Zucchini + + +# Bytes query +def test_bytes(test_store): + box = test_store.box(TestEntity) + + bytes_prop: Property = TestEntity.bytes + + id1 = box.put(TestEntity(bytes=bytes([9]))) + id2 = box.put(TestEntity(bytes=bytes([1,0]))) + id3 = box.put(TestEntity(bytes=bytes([0,1]))) + query = box.query(bytes_prop.greater_or_equal(bytes([0]))).build() + assert query.count() == 3 + query = box.query(bytes_prop.greater_or_equal(bytes([1]))).build() + assert query.count() == 2 + query = box.query(bytes_prop.greater_or_equal(bytes([9]))).build() + assert query.count() == 1 + + assert box.remove_all() == 3 + id1 = box.put(TestEntity(bytes=bytes([1,2,3,4]))) + id2 = box.put(TestEntity(bytes=bytes([5,6,7,8,9,10,11]))) + query = box.query(bytes_prop.equals(bytes([1,2,3,4]))).build() + assert query.count() == 1 + assert query.find()[0].id == id1 + + query = box.query(bytes_prop.greater_than(bytes([1,2,3,4]))).build() + assert query.count() == 1 + assert query.find()[0].id == id2 + + query = box.query(bytes_prop.greater_or_equal(bytes([1,2,3,4]))).build() + assert query.count() == 2 + + query = box.query(bytes_prop.greater_or_equal(bytes([0]))).build() + assert query.count() == 2 + + query = box.query(bytes_prop.less_than(bytes([5,6,7,8,9,10,11]))).build() + assert query.count() == 1 + assert query.find()[0].id == id1 + + query = box.query(bytes_prop.less_or_equal(bytes([5,6,7,8,9,10,11]))).build() + assert query.count() == 2 + + # bytes does not support not equals + with pytest.raises(AttributeError): + bytes_prop.not_equals(bytes([])) diff --git a/tests/test_store_options.py b/tests/test_store_options.py new file mode 100644 index 0000000..1e50e99 --- /dev/null +++ b/tests/test_store_options.py @@ -0,0 +1,80 @@ +from objectbox import Store +from objectbox.c import * # TODO ideally we wouldn't have to import c.py +from objectbox.store_options import StoreOptions +from tests.common import * + +def test_set_options(): + """ Test setting dummy values for each option. + Checks that Python types are correctly forwarded to C API. """ + + options = StoreOptions() + options.directory("test-db") + options.max_db_size_in_kb(8192) + options.max_data_size_in_kb(4096) + options.file_mode(755) + options.max_readers(10) + options.no_reader_thread_locals(False) + # options.model + # options.model_bytes + # options.model_bytes_direct + # options.validate_on_open_pages + # options.validate_on_open_kv + options.put_padding_mode(OBXPutPaddingMode_PaddingAutomatic) + options.read_schema(False) + options.use_previous_commit(False) + options.read_only(True) + options.debug_flags(OBXDebugFlags_LOG_TRANSACTIONS_READ) + options.add_debug_flags(OBXDebugFlags_LOG_CACHE_HITS) + options.async_max_queue_length(100) + options.async_throttle_at_queue_length(1024) + options.async_throttle_micros(1000) + options.async_max_in_tx_duration(1000) + options.async_max_in_tx_operations(20) + options.async_pre_txn_delay(500) + options.async_pre_txn_delay4(500, 700, 100) + options.async_post_txn_delay(500) + options.async_post_txn_delay5(500, 700, 100, False) + options.async_minor_refill_threshold(100) + options.async_minor_refill_max_count(500) + options.async_max_tx_pool_size(100) + options.async_object_bytes_max_cache_size(4096) + options.async_object_bytes_max_size_to_cache(4096) + # options.log_callback + # options.backup_restore + + assert options.get_directory() == "test-db" + assert options.get_max_db_size_in_kb() == 8192 + assert options.get_max_data_size_in_kb() == 4096 + assert options.get_debug_flags() == (OBXDebugFlags_LOG_TRANSACTIONS_READ | OBXDebugFlags_LOG_CACHE_HITS) + + del options + +def test_store_with_options(): + Store.remove_db_files("testdata") + remove_json_model_file() + + store = Store( + model=create_default_model(), + directory="testdata", + max_db_size_in_kb=1<<20, + max_data_size_in_kb=(1<<20)-(1<<10), + file_mode=int('664',8), + max_readers=126, + no_reader_thread_locals=True, + read_schema=True, + use_previous_commit=False, + read_only=False, + debug_flags=DebugFlags.LOG_TRANSACTIONS_READ|DebugFlags.LOG_TRANSACTIONS_WRITE, + async_max_queue_length=100, + async_throttle_at_queue_length=100, + async_throttle_micros=50000, + async_max_in_tx_duration=50000, + async_max_in_tx_operations=1000, + async_pre_txn_delay=100000, + async_post_txn_delay=100000, + async_minor_refill_threshold=10, + async_minor_refill_max_count=100, + async_object_bytes_max_cache_size=1<<20, + async_object_bytes_max_size_to_cache=100<<10 + ) + store.close() diff --git a/tests/test_transactions.py b/tests/test_transactions.py index 34d7c08..091d920 100644 --- a/tests/test_transactions.py +++ b/tests/test_transactions.py @@ -1,25 +1,23 @@ -import pytest import objectbox from tests.model import TestEntity -from tests.common import autocleanup, load_empty_test_objectbox, assert_equal +from tests.common import * -def test_transactions(): - ob = load_empty_test_objectbox() - box = objectbox.Box(ob, TestEntity) +def test_transactions(test_store): + box = test_store.box(TestEntity) assert box.is_empty() - with ob.write_tx(): - box.put(TestEntity("first")) - box.put(TestEntity("second")) + with test_store.write_tx(): + box.put(TestEntity(str="first")) + box.put(TestEntity(str="second")) assert box.count() == 2 try: - with ob.write_tx(): - box.put(TestEntity("third")) - box.put(TestEntity("fourth")) + with test_store.write_tx(): + box.put(TestEntity(str="third")) + box.put(TestEntity(str="fourth")) raise Exception("mission abort!") # exception must be propagated so this line must not execute @@ -32,10 +30,12 @@ def test_transactions(): # can't write in a read TX try: - with ob.read_tx(): - box.put(TestEntity("third")) + with test_store.read_tx(): + box.put(TestEntity(str="third")) # exception must be propagated so this line must not execute assert 0 except Exception as err: assert "Cannot start a write transaction inside a read only transaction" in str(err) + finally: + test_store.close() diff --git a/tests/test_userclass.py b/tests/test_userclass.py new file mode 100644 index 0000000..d8a0aa2 --- /dev/null +++ b/tests/test_userclass.py @@ -0,0 +1,44 @@ +from objectbox import * +from objectbox.model.idsync import sync_model + +def test_userclass(): + @Entity() + class Person: + id = Id() + firstName = String() + lastName = String() + + def __init__(self): + self.counter = 0 + + def fullname(self): + return f"{self.firstName} {self.lastName}" + + def tick(self): + self.counter += 1 + + model = Model() + model.entity(Person) + sync_model(model) + dbpath = "testdb" + Store.remove_db_files(dbpath) + + store = Store(model=model, directory=dbpath) + box = store.box(Person) + id_alice = box.put(Person(firstName="Alice", lastName="Adkinson")) + box.put(Person(firstName="Bob", lastName="Bowman")) + box.put(Person(firstName="Cydia", lastName="Cervesa")) + assert box.count() == 3 + alice = box.get(id_alice) + assert alice.fullname() == "Alice Adkinson" + assert alice.counter == 0 + alice.tick() + alice.tick() + assert alice.counter == 2 + + alice = box.get(id_alice) + assert alice.counter == 0 + + id_empty = box.put(Person()) + empty = box.get(id_empty) + assert empty.fullname() == " " diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..5799ba2 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,93 @@ +import sys +from datetime import timezone, datetime, timedelta + +import pytest + +from objectbox.utils import * + + +def test_date_value_to_int__basics(): + assert date_value_to_int(1234, 1000) == 1234 + assert date_value_to_int(1234, 1000000000) == 1234 + assert date_value_to_int(1234.0, 1000) == 1234000 # milliseconds + assert date_value_to_int(1234.0, 1000000000) == 1234000000000 # nanoseconds + dt = datetime.fromtimestamp(12345678) # May 1970; 1234 is too close to the epoch (special case for that below) + assert date_value_to_int(dt, 1000) == 12345678000 # milliseconds + + +def test_date_value_to_int__close_to_epoch(): + assert date_value_to_int(datetime.fromtimestamp(0, timezone.utc), 1000) == 0 + assert date_value_to_int(datetime.fromtimestamp(1234, timezone.utc), 1000) == 1234000 + assert date_value_to_int(datetime.fromtimestamp(0), 1000) == 0 + assert date_value_to_int(datetime.fromtimestamp(1234), 1000) == 1234000 + + # "Return the local date corresponding to the POSIX timestamp"; but not always!? Was -1 hour off with CEST: + dt0naive = datetime.fromtimestamp(0) + local_tz = datetime.now().astimezone().tzinfo + dt0local = dt0naive.replace(tzinfo=local_tz) + dt0utc = dt0local.astimezone(timezone.utc) + + # Print, don't assert... the result Seems to depend on the local timezone configuration!? + print("\nNaive:", dt0naive) # Seen: 1970-01-01 01:00:00 + print("Local:", dt0local) # Seen: 1970-01-01 01:00:00+02:00 + print("UTC:", dt0utc) # Seen: 1969-12-31 23:00:00+00:00 + print("Timestamp:", dt0utc.timestamp()) # Seen: -3600.0 + + dt = datetime.fromtimestamp(1234) + if sys.platform == "win32": + # On Windows, timestamp() seems to raise an OSError if the date is close to the epoch; see bug reports: + # https://github.com/python/cpython/issues/81708 and https://github.com/python/cpython/issues/94414 + try: + dt.timestamp() + assert False, "Expected OSError - Did Python on Windows get fixed?" + except OSError as e: + assert e.errno == 22 + else: + # Non-Windows platforms should work fine + assert dt.timestamp() == 1234 + + assert date_value_to_int(dt, 1000) == 1234000 # milliseconds + + +def test_date_value_to_int__timezone(): + # create datetime object for may 1st, 2000 + dt_utc = datetime(year=2000, month=5, day=1, hour=12, minute=30, second=45, microsecond=123456, tzinfo=timezone.utc) + dt_plus2 = datetime(year=2000, month=5, day=1, hour=14, minute=30, second=45, microsecond=123456, + tzinfo=timezone(offset=timedelta(hours=2))) + + # Demonstrate Python's semantic + assert dt_utc == dt_plus2 + assert dt_utc.timestamp() == dt_plus2.timestamp() + + # Actual test + expected: int = 957184245123 + assert date_value_to_int(dt_utc, 1000) == expected + assert date_value_to_int(dt_plus2, 1000) == expected + + +def test_date_value_to_int__naive(): + dt_naive = datetime(year=2000, month=5, day=1, hour=12, minute=30, second=45, microsecond=123456) + local_tz = datetime.now().astimezone().tzinfo + dt_local = datetime(year=2000, month=5, day=1, hour=12, minute=30, second=45, microsecond=123456, tzinfo=local_tz) + + # Demonstrate Python's semantic + assert dt_naive.astimezone(timezone.utc) == dt_local # naive lacks the TZ, so we can't compare directly + assert dt_naive.timestamp() == dt_local.timestamp() + + # Actual test + assert date_value_to_int(dt_naive, 1000) == date_value_to_int(dt_local, 1000) + + +def test_vector_distance_f32(): + """ Tests distance values between two vectors. """ + + a = np.array([3.4, 2.9, -10, 1.0], dtype=np.float32) + b = np.array([56., -1.2, 22, 2.0], dtype=np.float32) + + a_norm = a / np.linalg.norm(a) + b_norm = b / np.linalg.norm(b) + + assert vector_distance_f32(VectorDistanceType.EUCLIDEAN, a, b, 4) == pytest.approx(np.dot(b - a, b - a)) + assert vector_distance_f32(VectorDistanceType.COSINE, a, b, 4) == pytest.approx(1.0469311) + assert vector_distance_f32(VectorDistanceType.DOT_PRODUCT, a_norm, b_norm, 4) == pytest.approx(1.0469311) + assert vector_distance_f32(VectorDistanceType.DOT_PRODUCT_NON_NORMALIZED, a, b, 4) == pytest.approx(1.519307)