diff --git a/dbzero/setup.py b/dbzero/setup.py index 690f67bd..d536d2a8 100755 --- a/dbzero/setup.py +++ b/dbzero/setup.py @@ -10,7 +10,7 @@ setup( name='dbzero', - version='0.6.0', + version='0.6.1', description='DBZero community edition', packages=['dbzero'], python_requires='>=3.9', diff --git a/pyproject.toml b/pyproject.toml index 969594ab..54d785bd 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ requires = ['meson-python'] [project] name = 'dbzero' -version = '0.6.0' +version = '0.6.1' description = 'A state management system for Python 3.x that unifies your applications business logic, data persistence, and caching into a single, efficient layer.' readme = 'README.md' requires-python = '>=3.9' diff --git a/python_tests/test_index.py b/python_tests/test_index.py index 70dcfff1..87d40aa8 100755 --- a/python_tests/test_index.py +++ b/python_tests/test_index.py @@ -87,6 +87,17 @@ def test_index_can_store_nulls(db0_fixture): assert values == [2, 0, 3, 4, 1] +def test_find_with_negated_tag_and_empty_subquery_returns_empty(db0_fixture): + index = db0.index() + contact = MemoTestClass("not-overdue") + index.add(datetime(2026, 1, 2), contact) + + overdue_contacts = db0.find(MemoTestClass, index.select(None, datetime(2026, 1, 1))) + assert not overdue_contacts + + assert list(db0.find(MemoTestClass, db0.no("archived"), overdue_contacts)) == [] + + def test_index_can_sort_by_multiple_criteria(db0_fixture): index_1 = db0.index() index_2 = db0.index() diff --git a/python_tests/test_tag_fields.py b/python_tests/test_tag_fields.py index 250a2616..0d29944f 100755 --- a/python_tests/test_tag_fields.py +++ b/python_tests/test_tag_fields.py @@ -434,6 +434,73 @@ def __init__(self): assert list(db0.find(MixedRuntimeFields, "manual")) == [obj] +def test_passive_tag_field_then_regular_foreign_tag_preserves_regular_target(db0_fixture): + @db0.memo + class Scope: + pass + + @db0.memo + @db0.tag_fields("scope") + class PassiveRecord: + def __init__(self, scope): + self.scope = scope + + @db0.memo + class Error: + pass + + def add_error(scope): + error = Error() + db0.tags(error).add(db0.as_tag(scope)) + + scope = Scope() + + PassiveRecord(scope) + add_error(scope) + + assert len(list(db0.find(Error, db0.as_tag(scope)))) == 1 + + +def test_mixed_passive_regular_foreign_tag_remove_releases_regular_target(db0_fixture): + @db0.memo + class Scope: + pass + + @db0.memo + @db0.tag_fields("scope") + class PassiveRecord: + def __init__(self, scope): + self.scope = scope + + @db0.memo + class RegularTarget: + def __init__(self): + pass + + scope = Scope() + tag = db0.as_tag(scope) + PassiveRecord(scope) + + target = RegularTarget() + target_uuid = db0.uuid(target) + db0.tags(target).add(tag) + db0.commit() + + assert list(db0.find(RegularTarget, tag)) == [target] + assert db0.getrefcount(target) == 1 + + db0.tags(target).remove(tag) + db0.commit() + + assert list(db0.find(RegularTarget, tag)) == [] + assert db0.getrefcount(target) == 0 + + del target + db0.commit() + + assert not db0.exists(target_uuid) + + def test_tag_field_removal_preserves_manual_collision_semantics(db0_fixture): @db0.memo @db0.tag_fields("status") diff --git a/python_tests/test_tags.py b/python_tests/test_tags.py index f3b6e9fd..32ea2023 100755 --- a/python_tests/test_tags.py +++ b/python_tests/test_tags.py @@ -124,22 +124,41 @@ def test_passive_tags_can_be_added_and_removed_from_find_results(db0_fixture): assert {item.value for item in db0.find("other")} == {3} -def test_passive_first_then_regular_tag_remains_non_durable(db0_fixture): +def test_passive_first_then_regular_tag_upgrades_to_durable(db0_fixture): object_1 = MemoNoDefTags(1) db0.tags(object_1, passive=True).add("passive-tag") db0.commit() db0.tags(object_1).add("passive-tag") db0.commit() - assert db0.getrefcount(object_1) == 0 + assert db0.getrefcount(object_1) == 1 -def test_passive_first_then_regular_tag_in_same_transaction_remains_non_durable(db0_fixture): +def test_passive_first_then_regular_tag_in_same_transaction_upgrades_to_durable(db0_fixture): object_1 = MemoNoDefTags(1) db0.tags(object_1, passive=True).add("passive-tag") db0.tags(object_1).add("passive-tag") db0.commit() + assert db0.getrefcount(object_1) == 1 + + +def test_passive_first_then_regular_tag_remove_releases_upgrade(db0_fixture): + object_1 = MemoNoDefTags(1) + object_uuid = db0.uuid(object_1) + + db0.tags(object_1, passive=True).add("passive-tag") + db0.commit() + db0.tags(object_1).add("passive-tag") + db0.commit() + assert db0.getrefcount(object_1) == 1 + + db0.tags(object_1).remove("passive-tag") + db0.commit() assert db0.getrefcount(object_1) == 0 + del object_1 + db0.commit() + assert not db0.exists(object_uuid) + def test_regular_first_then_passive_tag_remains_durable(db0_fixture): object_1 = MemoNoDefTags(1) @@ -224,7 +243,7 @@ def test_passive_foreign_tag_remove_uses_regular_remove(db0_fixture): assert list(db0.find(MemoClassForTags, foreign_tag)) == [] -def test_passive_foreign_tag_first_then_regular_remains_non_durable(db0_fixture): +def test_passive_foreign_tag_first_then_regular_upgrades_to_durable(db0_fixture): foreign_tag_source = MemoScopedClass(2) foreign_tag = db0.as_tag(foreign_tag_source) db0.open("passive-long-tag-prefix", "rw") @@ -235,6 +254,29 @@ def test_passive_foreign_tag_first_then_regular_remains_non_durable(db0_fixture) db0.commit() db0.tags(local_object).add(foreign_tag) db0.commit() + assert db0.getrefcount(local_object) == 1 + + del local_object + db0.commit() + assert db0.exists(local_uuid) + assert [item.value for item in db0.find(MemoNoDefTags, foreign_tag)] == [1] + + +def test_passive_foreign_tag_first_then_regular_remove_releases_upgrade(db0_fixture): + foreign_tag_source = MemoScopedClass(2) + foreign_tag = db0.as_tag(foreign_tag_source) + db0.open("passive-long-tag-prefix", "rw") + local_object = MemoNoDefTags(1) + local_uuid = db0.uuid(local_object) + + db0.tags(local_object, passive=True).add(foreign_tag) + db0.commit() + db0.tags(local_object).add(foreign_tag) + db0.commit() + assert db0.getrefcount(local_object) == 1 + + db0.tags(local_object).remove(foreign_tag) + db0.commit() assert db0.getrefcount(local_object) == 0 del local_object diff --git a/src/dbzero/core/collections/b_index/bindex_interface.hpp b/src/dbzero/core/collections/b_index/bindex_interface.hpp index d40d9379..6d33788e 100755 --- a/src/dbzero/core/collections/b_index/bindex_interface.hpp +++ b/src/dbzero/core/collections/b_index/bindex_interface.hpp @@ -28,6 +28,17 @@ namespace db0::bindex::interface using bulkInsertUniquePtr = std::pair(*) (void *this_ptr, typename DefinitionT::Containers::IInputRange&, std::function *callback_ptr); + /** + * Heteromorphic inserts are for item types whose comparator/equality identity can match even + * when the stored bit pattern differs. On such duplicates, resolver_ptr decides whether the + * incoming representation should replace the stored one. + */ + template + using bulkInsertUniqueHeteromorphicPtr = std::pair(*) + (void *this_ptr, typename DefinitionT::Containers::IInputRange&, + std::function *callback_ptr, + typename DefinitionT::HeteromorphicResolverT *resolver_ptr); + template using insertPtr = void (*)(void *this_ptr, const item_t &item); template @@ -97,6 +108,8 @@ namespace db0::bindex::interface template struct BulkInsertUniqueFunctor {}; + template struct BulkInsertUniqueHeteromorphicFunctor {}; + template struct InsertFunctor {}; template struct BulkEraseFunctor {}; @@ -139,6 +152,17 @@ namespace db0::bindex::interface return static_cast(input).insert(index, callback_ptr); } + template + std::pair + insertHeteromorphicGenericImpl(void *self, typename DefinitionT::Containers::IInputRange &input, + std::function *callback_ptr, + typename DefinitionT::HeteromorphicResolverT *resolver_ptr) + { + ContainerT &index = *reinterpret_cast(self); + using InputRangeT = typename DefinitionT::template IContainerInputRange; + return static_cast(input).insertHeteromorphic(index, callback_ptr, resolver_ptr); + } + template std::size_t eraseGenericImpl(void *self, typename DefinitionT::Containers::IInputRange &input, std::function *callback_ptr) @@ -179,6 +203,18 @@ namespace db0::bindex::interface } }; + template + struct BulkInsertUniqueHeteromorphicFunctor> { + using CallbackT = std::function; + static std::pair + execute(void *this_ptr, typename DefinitionT::Containers::IInputRange &input, CallbackT *callback_ptr, + typename DefinitionT::HeteromorphicResolverT *resolver_ptr) + { + return insertHeteromorphicGenericImpl>( + this_ptr, input, callback_ptr, resolver_ptr); + } + }; + template struct InsertFunctor > { static void execute(void *this_ptr, const item_t &item) { reinterpret_cast*>(this_ptr)->insert(item); @@ -333,6 +369,20 @@ namespace db0::bindex::interface return insertGenericImpl(this_ptr, input, callback_ptr); } }; + + template + struct BulkInsertUniqueHeteromorphicFunctor> + { + using CallbackT = std::function; + static std::pair + execute(void *this_ptr, typename DefinitionT::Containers::IInputRange &input, CallbackT *callback_ptr, + typename DefinitionT::HeteromorphicResolverT *resolver_ptr) + { + using ContainerT = db0::v_sorted_sequence; + return insertHeteromorphicGenericImpl( + this_ptr, input, callback_ptr, resolver_ptr); + } + }; template struct InsertFunctor > { @@ -492,6 +542,18 @@ namespace db0::bindex::interface } }; + template + struct BulkInsertUniqueHeteromorphicFunctor> { + using CallbackT = std::function; + static std::pair + execute(void *this_ptr, typename DefinitionT::Containers::IInputRange &input, CallbackT *callback_ptr, + typename DefinitionT::HeteromorphicResolverT *resolver_ptr) + { + return insertHeteromorphicGenericImpl>( + this_ptr, input, callback_ptr, resolver_ptr); + } + }; + template struct InsertFunctor > { static void execute(void *this_ptr, const item_t &item) { reinterpret_cast*>(this_ptr)->insert(item); @@ -650,6 +712,18 @@ namespace db0::bindex::interface return insertGenericImpl>(this_ptr, input, callback_ptr); } }; + + template + struct BulkInsertUniqueHeteromorphicFunctor > { + using CallbackT = std::function; + static std::pair + execute(void *this_ptr, typename DefinitionT::Containers::IInputRange &input, CallbackT *callback_ptr, + typename DefinitionT::HeteromorphicResolverT *resolver_ptr) + { + return insertHeteromorphicGenericImpl>( + this_ptr, input, callback_ptr, resolver_ptr); + } + }; template struct InsertFunctor > { @@ -799,6 +873,19 @@ namespace db0::bindex::interface return insertGenericImpl >(this_ptr, input, callback_ptr); } }; + + template + struct BulkInsertUniqueHeteromorphicFunctor > + { + using CallbackT = std::function; + static std::pair + execute(void *this_ptr, typename DefinitionT::Containers::IInputRange &input, CallbackT *callback_ptr, + typename DefinitionT::HeteromorphicResolverT *resolver_ptr) + { + return insertHeteromorphicGenericImpl >( + this_ptr, input, callback_ptr, resolver_ptr); + } + }; template struct InsertFunctor > @@ -928,6 +1015,7 @@ namespace db0::bindex::interface : m_ref(ref) , m_ptr(m_ref.get()) , m_bulk_insert_unique_ptr(BulkInsertUniqueFunctor::execute) + , m_bulk_insert_unique_heteromorphic_ptr(BulkInsertUniqueHeteromorphicFunctor::execute) , m_insert_ptr(InsertFunctor::execute) , m_bulk_erase_ptr(BulkEraseFunctor::execute) , m_empty_ptr(EmptyFunctor::execute) @@ -956,6 +1044,14 @@ namespace db0::bindex::interface db0::bindex::GenericInputRange input(begin, end); return m_bulk_insert_unique_ptr(m_ptr, input, callback_ptr); } + + template + std::pair bulkInsertUniqueHeteromorphic(InputIterator begin, InputIterator end, + std::function *callback_ptr, typename DefinitionT::HeteromorphicResolverT *resolver_ptr) + { + db0::bindex::GenericInputRange input(begin, end); + return m_bulk_insert_unique_heteromorphic_ptr(m_ptr, input, callback_ptr, resolver_ptr); + } /** * Insert single item @@ -1068,6 +1164,7 @@ namespace db0::bindex::interface // pointer to actual data collection (persisted in m_ref) void *m_ptr = nullptr; bulkInsertUniquePtr m_bulk_insert_unique_ptr = nullptr; + bulkInsertUniqueHeteromorphicPtr m_bulk_insert_unique_heteromorphic_ptr = nullptr; insertPtr m_insert_ptr = nullptr; bulkErasePtr m_bulk_erase_ptr = nullptr; emptyPtr m_empty_ptr = nullptr; diff --git a/src/dbzero/core/collections/b_index/mb_index.hpp b/src/dbzero/core/collections/b_index/mb_index.hpp index fe323d4e..fd170338 100755 --- a/src/dbzero/core/collections/b_index/mb_index.hpp +++ b/src/dbzero/core/collections/b_index/mb_index.hpp @@ -3,6 +3,7 @@ #pragma once +#include #include #include "mb_index_def.hpp" @@ -10,6 +11,7 @@ #include "bindex_interface.hpp" #include "bindex_iterator.hpp" #include +#include namespace db0 @@ -33,9 +35,10 @@ namespace db0 using item_t = typename definition_t::item_t; using item_comp_t = typename definition_t::item_comp_t; - using interface_t = bindex::interface::Impl; + using interface_t = bindex::interface::Impl; using iterator_t = bindex::iterator::Impl; using CallbackT = std::function; + using HeteromorphicResolverT = typename definition_t::HeteromorphicResolverT; /** * 0 = empty_index (0 elements) @@ -172,6 +175,12 @@ namespace db0 */ bindex::type assessIndexType(std::size_t final_size) const { + if constexpr (std::is_same_v) { + if (final_size == 0) { + return bindex::type::empty; + } + return (final_size <= m_sv_limit) ? bindex::type::sorted_vector : bindex::type::bindex; + } // depending on final size assess resulting index type switch (final_size) { @@ -414,6 +423,49 @@ namespace db0 return m_interface.bulkInsertUnique(begin, end, callback_ptr); } + /** + * Like bulkInsertUnique, but duplicate identities with different bitwise values are resolved by callback. + */ + template + std::pair bulkInsertUniqueHeteromorphic(InputIterator begin, InputIterator end, + CallbackT *callback_ptr = nullptr, HeteromorphicResolverT *resolver_ptr = nullptr) + { + if (begin == end) { + return std::make_pair(0,0); + } + + bindex::type type = getIndexType(); + std::size_t unique_count = m_interface.countUnique(begin, end, m_sv_limit + 1); + if (unique_count == 0) { + return m_interface.bulkInsertUniqueHeteromorphic(begin, end, callback_ptr, resolver_ptr); + } + + bindex::type type_after_insertion = getIndexTypeAfterInsertion(unique_count); + if (type_after_insertion > type) { + if (type_after_insertion == bindex::type::bindex) { + morphTo(); + } else if (type_after_insertion == bindex::type::sorted_vector || resolver_ptr) { + morphTo(); + } else { + std::size_t diff = 0; + if (type_after_insertion == bindex::type::itty) { + assert(type==bindex::type::empty); + diff = morphToAndInsert(begin, end, callback_ptr); + } else if (type_after_insertion == bindex::type::array_2) { + diff = morphToAndInsert(begin, end, callback_ptr); + } else if (type_after_insertion==bindex::type::array_3) { + diff = morphToAndInsert(begin, end, callback_ptr); + } else if (type_after_insertion==bindex::type::array_4) { + diff = morphToAndInsert(begin, end, callback_ptr); + } else { + assert(false); + } + return std::make_pair(std::distance(begin, end), diff); + } + } + return m_interface.bulkInsertUniqueHeteromorphic(begin, end, callback_ptr, resolver_ptr); + } + bool isNull() const { return m_morph.isNull(); } diff --git a/src/dbzero/core/collections/b_index/mb_index_def.hpp b/src/dbzero/core/collections/b_index/mb_index_def.hpp index 429bf05a..0fcc05de 100755 --- a/src/dbzero/core/collections/b_index/mb_index_def.hpp +++ b/src/dbzero/core/collections/b_index/mb_index_def.hpp @@ -32,6 +32,7 @@ namespace db0::bindex using vector_t = v_sorted_vector; using bindex_t = v_bindex; using CallbackT = std::function; + using HeteromorphicResolverT = std::function; template class IContainerInputRange @@ -40,6 +41,8 @@ namespace db0::bindex virtual ~IContainerInputRange() = default; virtual std::pair insert(ContainerType&, CallbackT *callback_ptr) = 0; + virtual std::pair insertHeteromorphic(ContainerType&, + CallbackT *callback_ptr, HeteromorphicResolverT *resolver_ptr) = 0; virtual std::size_t erase(ContainerType&, CallbackT *callback_ptr) = 0; virtual std::size_t countNew(const ContainerType&, std::size_t max_count) = 0; virtual std::size_t countExisting(const ContainerType&, std::size_t max_count) = 0; diff --git a/src/dbzero/core/collections/b_index/mb_index_generic_input_range.hpp b/src/dbzero/core/collections/b_index/mb_index_generic_input_range.hpp index c819102d..43c21426 100755 --- a/src/dbzero/core/collections/b_index/mb_index_generic_input_range.hpp +++ b/src/dbzero/core/collections/b_index/mb_index_generic_input_range.hpp @@ -4,6 +4,7 @@ #pragma once #include "mb_index_def.hpp" +#include namespace db0::bindex @@ -20,6 +21,7 @@ namespace db0::bindex using vector_t = typename DefinitionT::vector_t; using bindex_t = typename DefinitionT::bindex_t; using CallbackT = typename DefinitionT::CallbackT; + using HeteromorphicResolverT = typename DefinitionT::HeteromorphicResolverT; IteratorT m_first, m_last; @@ -73,6 +75,33 @@ namespace db0::bindex return 0; } + template + std::pair indexInsertHeteromorphic(IndexContainer &index, + HeteromorphicResolverT *resolver_ptr) + { + using item_t = typename DefinitionT::item_t; + std::pair result(0, 0); + for (auto it = m_first; it != m_last; ++it) { + item_t incoming = *it; + ++result.first; + item_t old_value; + if (index.updateExisting(incoming, &old_value)) { + if (resolver_ptr && std::memcmp(&old_value, &incoming, sizeof(item_t)) != 0) { + item_t resolved; + if ((*resolver_ptr)(old_value, incoming, resolved)) { + index.updateExisting(resolved, nullptr); + } else { + index.updateExisting(old_value, nullptr); + } + } + continue; + } + THROWF(db0::InternalException) + << "Insert not supported in immutable container"; + } + return result; + } + public: GenericInputRange(const IteratorT &first, const IteratorT &last) : m_first(first) @@ -86,6 +115,12 @@ namespace db0::bindex return index.bulkInsertUnique(m_first, m_last, callback_ptr); } + virtual std::pair insertHeteromorphic( + bindex_t &index, CallbackT *callback_ptr, HeteromorphicResolverT *resolver_ptr) override + { + return index.bulkInsertUniqueHeteromorphic(m_first, m_last, callback_ptr, resolver_ptr); + } + virtual std::size_t erase(bindex_t &index, CallbackT *callback_ptr) override { using item_t = typename DefinitionT::item_t; @@ -107,6 +142,14 @@ namespace db0::bindex return result; } + virtual std::pair insertHeteromorphic( + vector_t &index, CallbackT *callback_ptr, HeteromorphicResolverT *resolver_ptr) override + { + std::pair result; + index.bulkInsertUniqueHeteromorphic(m_first, m_last, &result, callback_ptr, resolver_ptr); + return result; + } + virtual std::size_t erase(vector_t &index, CallbackT *callback_ptr) override { return index.bulkErase(m_first, m_last, callback_ptr); } @@ -124,6 +167,12 @@ namespace db0::bindex return arrayInsert(); } + virtual std::pair insertHeteromorphic( + array4_t &index, CallbackT *, HeteromorphicResolverT *resolver_ptr) override + { + return indexInsertHeteromorphic(index, resolver_ptr); + } + virtual std::size_t erase(array4_t&, CallbackT *) override { return arrayErase(); } @@ -141,6 +190,12 @@ namespace db0::bindex return arrayInsert(); } + virtual std::pair insertHeteromorphic( + array3_t &index, CallbackT *, HeteromorphicResolverT *resolver_ptr) override + { + return indexInsertHeteromorphic(index, resolver_ptr); + } + virtual std::size_t erase(array3_t&, CallbackT *) override { return arrayErase(); } @@ -158,6 +213,12 @@ namespace db0::bindex return arrayInsert(); } + virtual std::pair insertHeteromorphic( + array2_t &index, CallbackT *, HeteromorphicResolverT *resolver_ptr) override + { + return indexInsertHeteromorphic(index, resolver_ptr); + } + virtual std::size_t erase(array2_t&, CallbackT *) override { return arrayErase(); } @@ -175,6 +236,12 @@ namespace db0::bindex return arrayInsert(); } + virtual std::pair insertHeteromorphic( + itty_index_t &index, CallbackT *, HeteromorphicResolverT *resolver_ptr) override + { + return indexInsertHeteromorphic(index, resolver_ptr); + } + virtual std::size_t erase(itty_index_t&, CallbackT *) override { return arrayErase(); } @@ -214,6 +281,12 @@ namespace db0::bindex return arrayInsert(); } + virtual std::pair insertHeteromorphic( + empty_t &index, CallbackT *callback_ptr, HeteromorphicResolverT *) override + { + return insert(index, callback_ptr); + } + virtual std::size_t erase(empty_t&, CallbackT *) override { return arrayErase(); } @@ -228,4 +301,4 @@ namespace db0::bindex }; -} \ No newline at end of file +} diff --git a/src/dbzero/core/collections/b_index/v_bindex.hpp b/src/dbzero/core/collections/b_index/v_bindex.hpp index be1c2a71..7d93fc63 100755 --- a/src/dbzero/core/collections/b_index/v_bindex.hpp +++ b/src/dbzero/core/collections/b_index/v_bindex.hpp @@ -9,6 +9,7 @@ #include "v_bindex_joinable_const_iterator.hpp" #include "v_bindex_joinable_iterator.hpp" #include +#include #include namespace db0 @@ -43,6 +44,7 @@ namespace db0 using joinable_iterator = v_bindex_joinable_iterator; using DestroyF = std::function; using CallbackT = std::function; + using HeteromorphicResolverT = std::function; /** * Construct null instance @@ -195,6 +197,13 @@ namespace db0 return bulkInsert(begin_item, end_item, true, false, callback_ptr); } + template std::pair + bulkInsertUniqueHeteromorphic(iterator_t begin_item, iterator_t end_item, + CallbackT *callback_ptr = nullptr, HeteromorphicResolverT *resolver_ptr = nullptr) + { + return bulkInsert(begin_item, end_item, true, true, callback_ptr, resolver_ptr); + } + /** * Either inserts new items or updates existing with specific "update" lambda function */ @@ -212,7 +221,7 @@ namespace db0 */ template std::pair bulkInsert(iterator_t begin_item, iterator_t end_item, bool unique_only = false, bool update = false, - CallbackT *callback_ptr = nullptr) + CallbackT *callback_ptr = nullptr, HeteromorphicResolverT *resolver_ptr = nullptr) { assert(!update || unique_only); std::pair result(0, 0); @@ -225,7 +234,7 @@ namespace db0 } while (!data_heap.empty()) { insert_iterator insert_it(*this, data_heap.front()); - size_diff += insert_it.bulkInsert(data_heap, unique_only, update, callback_ptr); + size_diff += insert_it.bulkInsert(data_heap, unique_only, update, callback_ptr, resolver_ptr); } if (size_diff != 0) { this->modify().size += size_diff; @@ -625,7 +634,7 @@ namespace db0 * insert unique items only */ std::uint32_t bulkInsertUnique(heap &data) { - return bulkInsert(data, true, nullptr); + return bulkInsert(data, true, false, nullptr); } /** @@ -636,7 +645,7 @@ namespace db0 * @return number of items inserted */ std::uint32_t bulkInsert(heap &data, bool unique_only = false, - bool update = false, CallbackT *callback_ptr = nullptr) + bool update = false, CallbackT *callback_ptr = nullptr, HeteromorphicResolverT *resolver_ptr = nullptr) { assert(!update || unique_only); std::uint32_t result = 0; @@ -668,8 +677,11 @@ namespace db0 } else { // update existing item (value part) if not identical as existing one if (item_ptr && update && std::memcmp(item_ptr, &data.front(), sizeof(item_t)) != 0) { - auto at = m_data_buf->getItemIndex(item_ptr); - m_data_buf.modify().modifyItem(at) = data.front(); + item_t resolved = data.front(); + if (!resolver_ptr || (*resolver_ptr)(*item_ptr, data.front(), resolved)) { + auto at = m_data_buf->getItemIndex(item_ptr); + m_data_buf.modify().modifyItem(at) = resolved; + } } // duplicate item, remove all from insert heap data.pop_front_all(); diff --git a/src/dbzero/core/collections/full_text/FT_BaseIndex.cpp b/src/dbzero/core/collections/full_text/FT_BaseIndex.cpp index 1385cf5d..c5057dee 100755 --- a/src/dbzero/core/collections/full_text/FT_BaseIndex.cpp +++ b/src/dbzero/core/collections/full_text/FT_BaseIndex.cpp @@ -19,6 +19,10 @@ namespace db0 return value.isValid(); } + template <> bool is_valid(const UniqueRef &value) { + return value.isValid(); + } + template FT_BaseIndex::FT_BaseIndex(Memspace & memspace, VObjectCache &cache) : super_t(memspace, cache) @@ -38,14 +42,16 @@ namespace db0 } template - std::unique_ptr > + template + std::unique_ptr > FT_BaseIndex::makeIterator(IndexKeyT key, int direction) const { - return makeIterator(key, direction, std::vector { key }); + return makeIterator(key, direction, std::vector { key }); } template - std::unique_ptr > + template + std::unique_ptr > FT_BaseIndex::makeIterator(IndexKeyT key, int direction, std::vector &&index_key_sequence) const { @@ -54,22 +60,25 @@ namespace db0 if (!inverted_list_ptr) { return nullptr; } - return std::unique_ptr >( - new FT_IndexIterator( + return std::unique_ptr >( + new FT_IndexIterator( *inverted_list_ptr, direction, key, std::move(index_key_sequence), [this, key]() { return this->tryGetExistingInvertedList(key); }) ); } template - bool FT_BaseIndex::addIterator(FT_IteratorFactory &factory, IndexKeyT key) const + template + bool FT_BaseIndex::addIterator(FT_IteratorFactory &factory, + IndexKeyT key) const { - return addIterator(factory, key, std::vector { key }); + return addIterator(factory, key, std::vector { key }); } template - bool FT_BaseIndex::addIterator(FT_IteratorFactory &factory, IndexKeyT key, - std::vector &&index_key_sequence) const + template + bool FT_BaseIndex::addIterator(FT_IteratorFactory &factory, + IndexKeyT key, std::vector &&index_key_sequence) const { using ListT = typename super_t::ListT; auto inverted_list_ptr = this->tryGetExistingInvertedList(key); @@ -78,8 +87,8 @@ namespace db0 } // key inverted index - factory.add(std::unique_ptr >( - new FT_IndexIterator( + factory.add(std::unique_ptr >( + new FT_IndexIterator( *inverted_list_ptr, -1, key, std::move(index_key_sequence), [this, key]() { return this->tryGetExistingInvertedList(key); })) ); @@ -179,24 +188,64 @@ namespace db0 if (buf_begin == buf_end) { return; } - // sort the tags list and remove duplicate elements - std::sort(buf_begin, buf_end); - buf_end = std::unique(buf_begin, buf_end); + auto key_is_passive = [](IndexKeyT key) { + return FT_IndexKeyTraits::isPassive(key); + }; + std::function insert_callback; + if (insert_callback_ptr) { + insert_callback = [insert_callback_ptr](KeyT value) { + if (!FT_IndexValueTraits::isPassive(value)) { + (*insert_callback_ptr)(value); + } + }; + } + // NOTE: due to possible mixture of passive and regular tags + // we must identify uniform ranges by passive state. + auto compare_items = [&](typename TagValueList::const_reference lhs, + typename TagValueList::const_reference rhs) + { + if (lhs.first < rhs.first) { + return true; + } + if (rhs.first < lhs.first) { + return false; + } + auto lhs_passive = key_is_passive(lhs.first); + auto rhs_passive = key_is_passive(rhs.first); + if (lhs_passive != rhs_passive) { + return lhs_passive < rhs_passive; + } + return lhs.second < rhs.second; + }; + auto equal_items = [&](typename TagValueList::const_reference lhs, + typename TagValueList::const_reference rhs) + { + return lhs.first == rhs.first && + key_is_passive(lhs.first) == key_is_passive(rhs.first) && + lhs.second == rhs.second; + }; + + // Sort the tags list and remove duplicate elements. Passive and regular keys compare + // equal by logical tag, so keep passive state as an explicit range dimension. + std::sort(buf_begin, buf_end, compare_items); + buf_end = std::unique(buf_begin, buf_end, equal_items); TagRangesVector tag_ranges; - // Find ranges for all tags - // This vector will also effectively contain all unique tags + // Find ranges for all logical tags and passive states. tag_ranges.emplace_back(buf_begin); auto last_tag = buf_begin->first; + auto last_passive = key_is_passive(last_tag); for (auto it = buf_begin + 1; it != buf_end; ++it) { - if (it->first != last_tag) { + auto passive = key_is_passive(it->first); + if (it->first != last_tag || passive != last_passive) { tag_ranges.emplace_back(it); last_tag = it->first; + last_passive = passive; } } // Create inverted lists for tags and get corresponding iterators to them - std::vector::iterator> tag_index_its = index.bulkGetInvertedLists( + std::vector tag_index_its = index.bulkGetInvertedLists( TagIterator(tag_ranges.begin()), TagIterator(tag_ranges.end()), index_insert_callback_ptr @@ -208,21 +257,50 @@ namespace db0 for (std::size_t i = 0, n = tag_ranges.size() - 1; i < n; ++i) { auto range_first = tag_ranges[i], range_last = tag_ranges[i + 1]; // Either create new or pull existing inverted list - typename FT_BaseIndex::iterator &tag_index_it = tag_index_its[i]; + typename self_t::iterator &tag_index_it = tag_index_its[i]; assert((*tag_index_it).key == range_first->first); auto tag_index_ptr = index.getInvertedList(tag_index_it); auto old_addr = tag_index_ptr->getAddress(); auto old_map_value = addressOfMBIndex(*tag_index_ptr); // NOTICE: only unique items are retained in index // callback notified about unique items (objects) - auto *range_insert_callback_ptr = FT_IndexKeyPolicy::enableValueCallbacks( - range_first->first - ) ? insert_callback_ptr : nullptr; - std::pair stats = tag_index_ptr->bulkInsertUnique( - ValueIterator(range_first), - ValueIterator(range_last), - range_insert_callback_ptr - ); + // Use the actual key being inserted to decide whether values own references. + auto *range_insert_callback_ptr = !key_is_passive(range_first->first) && insert_callback + ? &insert_callback : nullptr; + std::pair stats; + if constexpr (FT_IndexValueTraits::canUpgradePassive()) { + if (range_insert_callback_ptr) { + typename super_t::ListT::HeteromorphicResolverT resolver = + [range_insert_callback_ptr](const KeyT &stored, const KeyT &incoming, KeyT &resolved) { + if (FT_IndexValueTraits::isPassive(stored) && + !FT_IndexValueTraits::isPassive(incoming)) + { + resolved = FT_IndexValueTraits::asOwning(incoming); + (*range_insert_callback_ptr)(resolved); + return true; + } + return false; + }; + stats = tag_index_ptr->bulkInsertUniqueHeteromorphic( + ValueIterator(range_first), + ValueIterator(range_last), + range_insert_callback_ptr, + &resolver + ); + } else { + stats = tag_index_ptr->bulkInsertUnique( + ValueIterator(range_first), + ValueIterator(range_last), + range_insert_callback_ptr + ); + } + } else { + stats = tag_index_ptr->bulkInsertUnique( + ValueIterator(range_first), + ValueIterator(range_last), + range_insert_callback_ptr + ); + } // This check is here because tag_index's location may have been changed by insert // We need to update pointer to tag_index (either address or type changed) @@ -251,6 +329,15 @@ namespace db0 return; } + std::function erase_callback; + if (erase_callback_ptr) { + erase_callback = [erase_callback_ptr](KeyT value) { + if (!FT_IndexValueTraits::isPassive(value)) { + (*erase_callback_ptr)(value); + } + }; + } + // Sort list and remove duplicate elements std::sort(buf_begin, buf_end); buf_end = std::unique(buf_begin, buf_end); @@ -262,7 +349,7 @@ namespace db0 return first_item.first != item.first; }); // instance collection by tag pointer - typename FT_BaseIndex::MapItemT item(first_item.first); + typename self_t::MapItemT item(first_item.first); auto it_list = index.find(item); if (it_list != index.end()) { auto stored_key = (*it_list).key; @@ -270,26 +357,32 @@ namespace db0 // we need to remember old type nd pointer because they may be modified by bulkErase operation auto old_addr = tag_index_ptr->getAddress(); auto old_map_value = addressOfMBIndex(*tag_index_ptr); - auto *range_erase_callback_ptr = FT_IndexKeyPolicy::enableValueCallbacks( - stored_key - ) ? erase_callback_ptr : nullptr; + // Removal requests may use a logically equivalent key without stored flags; + // use the requested key and stored value to decide whether refs are owned. + auto *range_erase_callback_ptr = !FT_IndexKeyTraits::isPassive( + first_item.first + ) && erase_callback ? &erase_callback : nullptr; std::size_t erased_count = tag_index_ptr->bulkErase( ValueIterator(buf_begin), ValueIterator(range_end), range_erase_callback_ptr ); auto new_map_value = addressOfMBIndex(*tag_index_ptr); - if (old_map_value != new_map_value) { - // Update list ptr in index + if (tag_index_ptr->empty()) { auto it = index.find(first_item.first); - if (tag_index_ptr->getIndexType() == db0::bindex::type::empty) { - // remove empty inverted list completely + if (it != index.end()) { index.erase(it); - // notify callback on index erased - if (index_erase_callback_ptr) { - (*index_erase_callback_ptr)(stored_key); - } - } else { + } + // notify callback on index erased + if (index_erase_callback_ptr) { + (*index_erase_callback_ptr)(stored_key); + } + // remove from cache since this instance has been removed + index.getVObjectCache().erase(old_addr); + } else if (old_map_value != new_map_value) { + // Update list ptr in index + auto it = index.find(first_item.first); + if (it != index.end()) { it.modifyItem().value = new_map_value; } // remove from cache since this instance has been relocated or removed @@ -503,7 +596,70 @@ namespace db0 template class FT_BaseIndex; template class FT_BaseIndex; + template bool FT_BaseIndex::addIterator( + FT_IteratorFactory &, db0::TagAddress) const; + template bool FT_BaseIndex::addIterator( + FT_IteratorFactory &, db0::TagAddress, std::vector &&) const; + template std::unique_ptr > + FT_BaseIndex::makeIterator(db0::TagAddress, int) const; + template std::unique_ptr > + FT_BaseIndex::makeIterator( + db0::TagAddress, int, std::vector &&) const; + + template bool FT_BaseIndex::addIterator( + FT_IteratorFactory &, db0::LongTagT) const; + template bool FT_BaseIndex::addIterator( + FT_IteratorFactory &, db0::LongTagT, std::vector &&) const; + template std::unique_ptr > + FT_BaseIndex::makeIterator(db0::LongTagT, int) const; + template std::unique_ptr > + FT_BaseIndex::makeIterator( + db0::LongTagT, int, std::vector &&) const; + + template class FT_BaseIndex; + template class FT_BaseIndex; + + template bool FT_BaseIndex::addIterator( + FT_IteratorFactory &, db0::TagAddress) const; + template bool FT_BaseIndex::addIterator( + FT_IteratorFactory &, db0::TagAddress, std::vector &&) const; + template std::unique_ptr > + FT_BaseIndex::makeIterator(db0::TagAddress, int) const; + template std::unique_ptr > + FT_BaseIndex::makeIterator( + db0::TagAddress, int, std::vector &&) const; + + template bool FT_BaseIndex::addIterator( + FT_IteratorFactory &, db0::LongTagT) const; + template bool FT_BaseIndex::addIterator( + FT_IteratorFactory &, db0::LongTagT, std::vector &&) const; + template std::unique_ptr > + FT_BaseIndex::makeIterator(db0::LongTagT, int) const; + template std::unique_ptr > + FT_BaseIndex::makeIterator( + db0::LongTagT, int, std::vector &&) const; + template class FT_BaseIndex; template class FT_BaseIndex; + + template bool FT_BaseIndex::addIterator( + FT_IteratorFactory &, std::uint64_t) const; + template bool FT_BaseIndex::addIterator( + FT_IteratorFactory &, std::uint64_t, std::vector &&) const; + template std::unique_ptr > + FT_BaseIndex::makeIterator(std::uint64_t, int) const; + template std::unique_ptr > + FT_BaseIndex::makeIterator( + std::uint64_t, int, std::vector &&) const; + + template bool FT_BaseIndex::addIterator( + FT_IteratorFactory &, db0::LongTagT) const; + template bool FT_BaseIndex::addIterator( + FT_IteratorFactory &, db0::LongTagT, std::vector &&) const; + template std::unique_ptr > + FT_BaseIndex::makeIterator(db0::LongTagT, int) const; + template std::unique_ptr > + FT_BaseIndex::makeIterator( + db0::LongTagT, int, std::vector &&) const; } diff --git a/src/dbzero/core/collections/full_text/FT_BaseIndex.hpp b/src/dbzero/core/collections/full_text/FT_BaseIndex.hpp index bde3621a..2520a389 100755 --- a/src/dbzero/core/collections/full_text/FT_BaseIndex.hpp +++ b/src/dbzero/core/collections/full_text/FT_BaseIndex.hpp @@ -9,36 +9,70 @@ #include "FT_ANDIterator.hpp" #include "FT_ORXIterator.hpp" #include +#include #include "LongTag.hpp" #include namespace db0 { + // Traits for index-key metadata carried alongside the logical key. template - struct FT_IndexKeyPolicy + struct FT_IndexKeyTraits { - static bool enableValueCallbacks(const IndexKeyT &) { - return true; + static bool isPassive(const IndexKeyT &) { + return false; } }; template <> - struct FT_IndexKeyPolicy + struct FT_IndexKeyTraits { - static bool enableValueCallbacks(db0::TagAddress tag_addr) { - return !tag_addr.isPassive(); + static bool isPassive(db0::TagAddress tag_addr) { + return tag_addr.isPassive(); } }; template <> - struct FT_IndexKeyPolicy + struct FT_IndexKeyTraits + { + static bool isPassive(const db0::LongTagT &tag_addr) { + return db0::isPassiveLongTag(tag_addr); + } + }; + + template + struct FT_IndexValueTraits { - static bool enableValueCallbacks(const db0::LongTagT &tag_addr) { - return !db0::isPassiveLongTag(tag_addr); + static bool isPassive(const KeyT &) { + return false; + } + + static constexpr bool canUpgradePassive() { + return false; + } + + static KeyT asOwning(const KeyT &value) { + return value; } }; + template <> + struct FT_IndexValueTraits + { + static bool isPassive(db0::UniqueRef ref) { + return ref.isPassive(); + } + + static constexpr bool canUpgradePassive() { + return true; + } + + static db0::UniqueRef asOwning(db0::UniqueRef ref) { + return ref.asOwning(); + } + }; + // FT_BaseIndex provides common API for managing tag/type inverted lists // @tparam IndexKeyT the tag / element's key type template @@ -59,18 +93,23 @@ namespace db0 * Collect iterator associated with a specific key (e.g. tag/type) * @return false if no iterator collected (e.g. no such key) */ - bool addIterator(FT_IteratorFactory &, IndexKeyT key) const; + template + bool addIterator(FT_IteratorFactory &, IndexKeyT key) const; // index_key_sequence is optional serialization metadata for nested // composite-tag queries; it is the root-to-leaf tag key path. - bool addIterator(FT_IteratorFactory &, IndexKeyT key, std::vector &&index_key_sequence) const; + template + bool addIterator(FT_IteratorFactory &, IndexKeyT key, + std::vector &&index_key_sequence) const; /** * @param key either tag or class identifier */ - std::unique_ptr > makeIterator(IndexKeyT key, int direction = -1) const; + template + std::unique_ptr > makeIterator(IndexKeyT key, int direction = -1) const; // See addIterator overload above: index_key_sequence is not used for // lookup, only to serialize enough context to reopen a nested tag path. - std::unique_ptr > makeIterator(IndexKeyT key, int direction, + template + std::unique_ptr > makeIterator(IndexKeyT key, int direction, std::vector &&index_key_sequence) const; /** @@ -145,12 +184,30 @@ namespace db0 template struct ValueHash { std::size_t operator()(const std::pair &value) const { - return std::hash()(value.first) ^ std::hash()(value.second); + return std::hash()(value.first) + ^ std::hash()(value.second) + ^ (FT_IndexKeyTraits::isPassive(value.first) ? 1 : 0) + ^ (FT_IndexValueTraits::isPassive(value.second) ? 2 : 0); + } + }; + + template struct ValueEqual + { + bool operator()(const std::pair &lhs, + const std::pair &rhs) const + { + return lhs.first == rhs.first && + FT_IndexKeyTraits::isPassive(lhs.first) == + FT_IndexKeyTraits::isPassive(rhs.first) && + lhs.second == rhs.second && + FT_IndexValueTraits::isPassive(lhs.second) == + FT_IndexValueTraits::isPassive(rhs.second); } }; - std::unordered_set, ValueHash > m_values; - std::unordered_set, ValueHash > m_value_refs; + std::unordered_set, ValueHash, ValueEqual > m_values; + std::unordered_set, ValueHash, ValueEqual > + m_value_refs; // a set of keys for which all operations should be reverted / ignored std::unordered_set m_reverted; @@ -331,6 +388,8 @@ namespace db0 extern template class FT_BaseIndex; extern template class FT_BaseIndex; extern template class FT_BaseIndex; + extern template class FT_BaseIndex; + extern template class FT_BaseIndex; extern template class FT_BaseIndex; extern template class FT_BaseIndex; diff --git a/src/dbzero/core/collections/full_text/FT_Serialization.hpp b/src/dbzero/core/collections/full_text/FT_Serialization.hpp index c42a7d92..41909ad0 100755 --- a/src/dbzero/core/collections/full_text/FT_Serialization.hpp +++ b/src/dbzero/core/collections/full_text/FT_Serialization.hpp @@ -16,6 +16,7 @@ #include "FT_FixedKeyIterator.hpp" #include #include +#include #include #include #include @@ -46,9 +47,12 @@ namespace db0 // detect underlying index type (complex type) auto _iter = iter; auto index_type_id = db0::serial::read(_iter, end); - if (index_type_id == db0::MorphingBIndex::getSerialTypeId()) { + if (index_type_id == db0::MorphingBIndex::getSerialTypeId() || + index_type_id == db0::MorphingBIndex::getSerialTypeId()) { auto key_type_id = db0::serial::read(_iter, end); auto index_key_type_id = db0::serial::read(_iter, end); + const bool unique_ref_index = + index_type_id == db0::MorphingBIndex::getSerialTypeId(); if (key_type_id == db0::serial::typeId()) { if constexpr (std::is_same_v) { if (index_key_type_id == db0::serial::typeId() || @@ -65,6 +69,9 @@ namespace db0 if constexpr (std::is_same_v) { if (index_key_type_id == db0::serial::typeId() || index_key_type_id == db0::serial::typeId()) { + if (unique_ref_index) { + return deserializeFT_IndexIterator, KeyT, db0::TagAddress>(workspace, iter, end); + } return deserializeFT_IndexIterator, KeyT, db0::TagAddress>(workspace, iter, end); } else { THROWF(db0::InternalException) << "Unsupported index key type ID: " << index_key_type_id diff --git a/src/dbzero/core/collections/vector/v_sorted_vector.hpp b/src/dbzero/core/collections/vector/v_sorted_vector.hpp index f6ea765b..c74005bd 100755 --- a/src/dbzero/core/collections/vector/v_sorted_vector.hpp +++ b/src/dbzero/core/collections/vector/v_sorted_vector.hpp @@ -76,6 +76,7 @@ DB0_PACKED_BEGIN using joinable_const_iterator = db0::joinable_const_iterator; using DestroyF = std::function; using CallbackT = std::function; + using HeteromorphicResolverT = std::function; static std::size_t measure(const o_sv_container &other) { return other.sizeOf(); @@ -307,6 +308,83 @@ DB0_PACKED_BEGIN return unique_count; } + template size_t bulkInsertUniqueHeteromorphic(iterator_t data_begin, iterator_t data_end, + std::size_t data_size, CallbackT *callback_ptr = nullptr, HeteromorphicResolverT *resolver_ptr = nullptr) + { + if (!resolver_ptr) { + return bulkInsertUnique(data_begin, data_end, data_size, callback_ptr); + } + + std::size_t unique_count = 0; + assert(m_size + data_size <= m_capacity); + SortedArray data_buf(begin(), end()); + heap s_heap(data_size); + { + auto it = data_begin; + while (it != data_end) { + s_heap.insert(*it); + ++it; + } + } + data_t *item = const_cast(data_buf.m_begin); + while (!s_heap.empty() && item != data_buf.m_end) { + item = const_cast(data_buf.join(item, s_heap.front(), 1)); + if (item != data_buf.m_end) { + if (data_buf.m_comp(s_heap.front(), *item)) { +#ifdef __linux__ + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wclass-memaccess" +#endif + memmove((item + 1), item, (data_buf.m_end - item) * sizeof(data_t)); +#ifdef __linux__ + #pragma GCC diagnostic pop +#endif + ++(data_buf.m_end); + ++(this->m_size); + ++unique_count; + if (callback_ptr) { + (*callback_ptr)(s_heap.front()); + } + *item = s_heap.front(); + } else if (std::memcmp(item, &s_heap.front(), sizeof(data_t)) != 0) { + data_t resolved; + if ((*resolver_ptr)(*item, s_heap.front(), resolved)) { + *item = resolved; + } + } + s_heap.pop_front(); + } + } + if (!s_heap.empty()) { + item = const_cast(data_buf.m_end); + *item = s_heap.front(); + ++unique_count; + if (callback_ptr) { + (*callback_ptr)(s_heap.front()); + } + s_heap.pop_front(); + ++(this->m_size); + while (!s_heap.empty()) { + if (data_buf.m_comp(*item, s_heap.front())) { + ++item; + *item = s_heap.front(); + ++(this->m_size); + ++unique_count; + if (callback_ptr) { + (*callback_ptr)(s_heap.front()); + } + } else if (std::memcmp(item, &s_heap.front(), sizeof(data_t)) != 0) { + data_t resolved; + if ((*resolver_ptr)(*item, s_heap.front(), resolved)) { + *item = resolved; + } + } + s_heap.pop_front(); + } + } + return unique_count; + } + /** * insert new / update existing * @return number of unique items inserted @@ -698,6 +776,7 @@ DB0_PACKED_END using joinable_const_iterator = typename c_type::joinable_const_iterator; using DestroyF = std::function; using CallbackT = std::function; + using HeteromorphicResolverT = typename c_type::HeteromorphicResolverT; v_sorted_vector() = default; @@ -937,6 +1016,24 @@ DB0_PACKED_END return addr_change; } + template bool bulkInsertUniqueHeteromorphic(iterator_t data_begin, iterator_t data_end, + std::pair *result, + CallbackT *callback_ptr = nullptr, HeteromorphicResolverT *resolver_ptr = nullptr, + std::optional max_size = {}) + { + std::size_t data_size = std::distance(data_begin, data_end); + if (result) { + result->first = static_cast(data_size); + } + bool addr_change = growVector((*this)->m_size + data_size, max_size); + std::size_t unique_count = this->modify().bulkInsertUniqueHeteromorphic( + data_begin, data_end, data_size, callback_ptr, resolver_ptr); + if (result) { + result->second = static_cast(unique_count); + } + return addr_change; + } + /** * insert new / update existing * @return true if object relocated diff --git a/src/dbzero/core/memory/Address.hpp b/src/dbzero/core/memory/Address.hpp index 1927bb4f..de09f589 100755 --- a/src/dbzero/core/memory/Address.hpp +++ b/src/dbzero/core/memory/Address.hpp @@ -199,8 +199,12 @@ DB0_PACKED_BEGIN class DB0_PACKED_ATTR TagAddress { public: + // TagAddress is used by the short-tag index, whose key space contains + // both real memory addresses and packed non-address tags. EnumValue_UID + // tags reserve bit 63. Address-backed tags use low 50 bits, so bit 50 + // is available as the passive marker without colliding with addresses. static constexpr std::uint64_t ENUM_BIT = 1ULL << 63; - static constexpr std::uint64_t PASSIVE_BIT = 1ULL << 62; + static constexpr std::uint64_t PASSIVE_BIT = 1ULL << 50; static constexpr std::uint64_t ADDRESS_MASK = (1ULL << 50) - 1; TagAddress() = default; @@ -231,6 +235,9 @@ DB0_PACKED_BEGIN inline TagAddress asPassive() const { auto regular_value = regularValue(m_value); + // Non-address tag encodings, such as enum and field-def tags, are + // not eligible for passive storage. Returning them unchanged keeps + // their packed identity intact. if ((regular_value & ~ADDRESS_MASK) != 0) { return *this; } diff --git a/src/dbzero/core/memory/UniqueRef.cpp b/src/dbzero/core/memory/UniqueRef.cpp new file mode 100644 index 00000000..59685cf0 --- /dev/null +++ b/src/dbzero/core/memory/UniqueRef.cpp @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright (c) 2025 DBZero Software sp. z o.o. + +#include "UniqueRef.hpp" + +#include + +namespace db0 + +{ + + UniqueRef::UniqueRef(UniqueAddress address, bool passive) + : m_value(address.getValue() | (passive ? PASSIVE_BIT : 0)) + { + if (address.getOffset() >= OFFSET_MAX) { + throw std::out_of_range("UniqueRef address offset exceeds 49-bit range"); + } + } + +} diff --git a/src/dbzero/core/memory/UniqueRef.hpp b/src/dbzero/core/memory/UniqueRef.hpp new file mode 100644 index 00000000..24620c8e --- /dev/null +++ b/src/dbzero/core/memory/UniqueRef.hpp @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright (c) 2025 DBZero Software sp. z o.o. + +#pragma once + +#include +#include +#include +#include +#include + +namespace db0 + +{ + +DB0_PACKED_BEGIN + /** + * UniqueRef stores an inverted-list reference in the same 64-bit footprint as UniqueAddress, + * with one high bit reserved to mark passive references. + * + * Tag-field indexing can point at an object without owning its lifetime, while explicit tags + * must keep the target object alive. Keeping that passive/owning bit next to the indexed value + * lets the index choose reference-counting behavior without widening stored inverted-list + * entries. Comparisons intentionally ignore the passive bit so both forms still identify the + * same logical object address. + */ + class DB0_PACKED_ATTR UniqueRef + { + public: + static constexpr std::uint64_t PASSIVE_BIT = 1ULL << 63; + static constexpr std::uint64_t VALUE_MASK = ~PASSIVE_BIT; + static constexpr std::uint64_t OFFSET_MAX = 1ULL << 49; + + UniqueRef() = default; + UniqueRef(UniqueAddress address, bool passive = false); + + static inline UniqueRef fromValue(std::uint64_t value); + + inline bool isValid() const; + inline bool isPassive() const; + inline bool isOwning() const; + inline UniqueRef asPassive() const; + inline UniqueRef asOwning() const; + inline std::uint64_t getValue() const; + inline UniqueAddress asUniqueAddress() const; + inline Address getAddress() const; + + inline operator UniqueAddress() const; + + inline bool operator==(const UniqueRef& other) const; + inline bool operator!=(const UniqueRef& other) const; + inline bool operator<(const UniqueRef& other) const; + inline bool operator<=(const UniqueRef& other) const; + inline bool operator>(const UniqueRef& other) const; + inline bool operator>=(const UniqueRef& other) const; + + inline friend std::ostream &operator<<(std::ostream &os, const UniqueRef &ref); + + private: + std::uint64_t m_value = 0; + + explicit inline UniqueRef(std::uint64_t value); + }; +DB0_PACKED_END + + inline UniqueRef UniqueRef::fromValue(std::uint64_t value) + { + return UniqueRef(value); + } + + inline bool UniqueRef::isValid() const + { + return (m_value & VALUE_MASK) != 0; + } + + inline bool UniqueRef::isPassive() const + { + return (m_value & PASSIVE_BIT) != 0; + } + + inline bool UniqueRef::isOwning() const + { + return !isPassive(); + } + + inline UniqueRef UniqueRef::asPassive() const + { + return UniqueRef(m_value | PASSIVE_BIT); + } + + inline UniqueRef UniqueRef::asOwning() const + { + return UniqueRef(m_value & VALUE_MASK); + } + + inline std::uint64_t UniqueRef::getValue() const + { + return m_value; + } + + inline UniqueAddress UniqueRef::asUniqueAddress() const + { + return UniqueAddress::fromValue(m_value & VALUE_MASK); + } + + inline Address UniqueRef::getAddress() const + { + return asUniqueAddress().getAddress(); + } + + inline UniqueRef::operator UniqueAddress() const + { + return asUniqueAddress(); + } + + inline bool UniqueRef::operator==(const UniqueRef& other) const + { + return (m_value & VALUE_MASK) == (other.m_value & VALUE_MASK); + } + + inline bool UniqueRef::operator!=(const UniqueRef& other) const + { + return !(*this == other); + } + + inline bool UniqueRef::operator<(const UniqueRef& other) const + { + return (m_value & VALUE_MASK) < (other.m_value & VALUE_MASK); + } + + inline bool UniqueRef::operator<=(const UniqueRef& other) const + { + return (m_value & VALUE_MASK) <= (other.m_value & VALUE_MASK); + } + + inline bool UniqueRef::operator>(const UniqueRef& other) const + { + return (m_value & VALUE_MASK) > (other.m_value & VALUE_MASK); + } + + inline bool UniqueRef::operator>=(const UniqueRef& other) const + { + return (m_value & VALUE_MASK) >= (other.m_value & VALUE_MASK); + } + + inline std::ostream &operator<<(std::ostream &os, const UniqueRef &ref) + { + os << ref.m_value; + return os; + } + + inline UniqueRef::UniqueRef(std::uint64_t value) + : m_value(value) + { + } + +} + +namespace std + +{ + + template <> struct hash { + inline std::size_t operator()(const db0::UniqueRef &ref) const noexcept; + }; + + inline std::size_t hash::operator()(const db0::UniqueRef &ref) const noexcept + { + return std::hash()(ref.asUniqueAddress().getValue()); + } + +} diff --git a/src/dbzero/core/serialization/Serializable.hpp b/src/dbzero/core/serialization/Serializable.hpp index 017d4087..bf32c09f 100755 --- a/src/dbzero/core/serialization/Serializable.hpp +++ b/src/dbzero/core/serialization/Serializable.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -144,6 +145,7 @@ namespace db0::serial using TypeList = std::tuple< std::int8_t, std::int16_t, std::int32_t, std::int64_t, std::uint8_t, std::uint16_t, std::uint32_t, std::uint64_t, db0::Address, db0::UniqueAddress, + db0::UniqueRef, float, double, std::string, db0::TagAddress>; // compile error: binary expression in operand of fold-expression std::apply([&](auto... type) { diff --git a/src/dbzero/object_model/tags/TagIndex.cpp b/src/dbzero/object_model/tags/TagIndex.cpp index 4dcea6e8..958297e3 100755 --- a/src/dbzero/object_model/tags/TagIndex.cpp +++ b/src/dbzero/object_model/tags/TagIndex.cpp @@ -188,10 +188,10 @@ namespace db0::object_model } using IterableSequence = TagMakerSequence; - ActiveValueT active_key = { UniqueAddress(), nullptr }; - auto &batch_op_short = getBatchOperationShort(memo_ptr, active_key, false); + ActiveValueT active_key = { UniqueRef(), nullptr }; + auto &batch_op_short = getBatchOperationShortTag(memo_ptr, active_key, passive); // since it's less common, defer initialization until first occurence - db0::FT_BaseIndex::BatchOperationBuilder *batch_op_long_ptr = nullptr; + TagBaseIndexLongT::BatchOperationBuilder *batch_op_long_ptr = nullptr; auto &type_manager = LangToolkit::getTypeManager(); for (std::size_t i = 0; i < nargs; ++i) { ObjectPtr arg = args[i]; @@ -212,7 +212,7 @@ namespace db0::object_model // sequence (pair) may represent a single long tag if (isLongTag(arg)) { if (!batch_op_long_ptr) { - batch_op_long_ptr = &getBatchOperationLong(memo_ptr, active_key); + batch_op_long_ptr = &getBatchOperationLongTag(memo_ptr, active_key, passive); } auto tag = makeLongTagFromSequence(tag_sequence); if (passive) { @@ -234,7 +234,7 @@ namespace db0::object_model } else { // must try adding as a long tag (item from a foreign scope) if (!batch_op_long_ptr) { - batch_op_long_ptr = &getBatchOperationLong(memo_ptr, active_key); + batch_op_long_ptr = &getBatchOperationLongTag(memo_ptr, active_key, passive); } auto long_tag = getLongTag(arg); if (passive) { @@ -271,7 +271,7 @@ namespace db0::object_model } } - PassiveTag TagIndex::preparePassiveTag(ObjectPtr arg, std::uint32_t source_id) const + TagIndex::PassiveTag TagIndex::preparePassiveTag(ObjectPtr arg, std::uint32_t source_id) const { auto type_id = LangToolkit::getTypeManager().getTypeId(arg); bool inc_ref = type_id == TypeId::STRING; @@ -286,13 +286,13 @@ namespace db0::object_model void TagIndex::add(ObjectPtr memo_ptr, PassiveTag &tag) { assert(tag.hasTag()); - ActiveValueT active_key = { UniqueAddress(), nullptr }; + ActiveValueT active_key = { UniqueRef(), nullptr }; if (tag.is_long()) { - auto &batch = getBatchOperationLong(memo_ptr, active_key); + auto &batch = getBatchOperationLongTag(memo_ptr, active_key, true); auto passive_tag = asPassiveLongTag(tag.long_tag()); batch->addTag(active_key, passive_tag); } else { - auto &batch = getBatchOperationShort(memo_ptr, active_key, false); + auto &batch = getBatchOperationShortTag(memo_ptr, active_key, true); auto short_tag = tag.short_tag(); auto passive_tag = short_tag.asPassive(); batch->addTag(active_key, passive_tag); @@ -307,35 +307,48 @@ namespace db0::object_model void TagIndex::remove(ObjectPtr memo_ptr, const PassiveTag &tag) { - ActiveValueT active_key = { UniqueAddress(), nullptr }; + ActiveValueT active_key = { UniqueRef(), nullptr }; if (tag.is_long()) { - auto &batch = getBatchOperationLong(memo_ptr, active_key); + auto &batch = getBatchOperationLongTag(memo_ptr, active_key); batch->removeTag(active_key, tag.long_tag()); } else { - auto &batch = getBatchOperationShort(memo_ptr, active_key, false); + auto &batch = getBatchOperationShortTag(memo_ptr, active_key); batch->removeTag(active_key, tag.short_tag()); } m_mutation_log->onDirty(); } - FT_BaseIndex::BatchOperationBuilder & + TagIndex::TagBaseIndexShortT::BatchOperationBuilder & + TagIndex::getBatchOperationType(ObjectPtr memo_ptr, ActiveValueT &result) const + { + return getBatchOperation(memo_ptr, m_base_index_short, m_batch_op_types, result); + } + + TagIndex::TagBaseIndexShortT::BatchOperationBuilder & + TagIndex::getBatchOperationShortTag(ObjectPtr memo_ptr, ActiveValueT &result, bool passive) const + { + return getBatchOperation(memo_ptr, m_base_index_short, m_batch_op_short, result, passive); + } + + TagIndex::TagBaseIndexLongT::BatchOperationBuilder & + TagIndex::getBatchOperationLongTag(ObjectPtr memo_ptr, ActiveValueT &result, bool passive) const + { + return getBatchOperation(memo_ptr, m_base_index_long, m_batch_op_long, result, passive); + } + + TagIndex::TagBaseIndexShortT::BatchOperationBuilder & TagIndex::getBatchOperationShort(ObjectPtr memo_ptr, ActiveValueT &result, bool is_type) const { if (is_type) { - return getBatchOperation( - memo_ptr, m_base_index_short, m_batch_op_types, result - ); - } else { - return getBatchOperation( - memo_ptr, m_base_index_short, m_batch_op_short, result - ); + return getBatchOperationType(memo_ptr, result); } + return getBatchOperationShortTag(memo_ptr, result); } - db0::FT_BaseIndex::BatchOperationBuilder & + TagIndex::TagBaseIndexLongT::BatchOperationBuilder & TagIndex::getBatchOperationLong(ObjectPtr memo_ptr, ActiveValueT &result) const { - return getBatchOperation(memo_ptr, m_base_index_long, m_batch_op_long, result); + return getBatchOperationLongTag(memo_ptr, result); } void TagIndex::addTag(ObjectPtr memo_ptr, Address tag_addr, bool is_type) { @@ -344,15 +357,21 @@ namespace db0::object_model void TagIndex::addTag(ObjectPtr memo_ptr, ShortTagT tag, bool is_type) { - ActiveValueT active_key = { UniqueAddress(), nullptr }; - auto &batch_operation = getBatchOperationShort(memo_ptr, active_key, is_type); - batch_operation->addTags(active_key, TagPtrSequence(&tag, &tag + 1)); + if (is_type) { + ActiveValueT active_key = { UniqueAddress(), nullptr }; + auto &batch_operation = getBatchOperationType(memo_ptr, active_key); + batch_operation->addTags(active_key, TagPtrSequence(&tag, &tag + 1)); + } else { + ActiveValueT active_key = { UniqueRef(), nullptr }; + auto &batch_operation = getBatchOperationShortTag(memo_ptr, active_key); + batch_operation->addTags(active_key, TagPtrSequence(&tag, &tag + 1)); + } m_mutation_log->onDirty(); } void TagIndex::addTag(ObjectPtr memo_ptr, LongTagT tag) { - ActiveValueT active_key = { UniqueAddress(), nullptr }; + ActiveValueT active_key = { UniqueRef(), nullptr }; auto &batch_operation = getBatchOperationLong(memo_ptr, active_key); batch_operation->addTags(active_key, TagPtrSequence(&tag, &tag + 1)); m_mutation_log->onDirty(); @@ -411,7 +430,7 @@ namespace db0::object_model void TagIndex::removeTypeTag(UniqueAddress obj_addr, Address tag_addr) { auto &batch_operation = getBatchOperation(m_base_index_short, m_batch_op_types); - batch_operation->removeTag({ obj_addr, nullptr }, ShortTagT::fromAddress(tag_addr)); + batch_operation->removeTag({ UniqueRef(obj_addr), nullptr }, ShortTagT::fromAddress(tag_addr)); m_mutation_log->onDirty(); } @@ -423,8 +442,8 @@ namespace db0::object_model using IterableSequence = TagMakerSequence; ActiveValueT active_key = { UniqueAddress(), nullptr }; - db0::FT_BaseIndex::BatchOperationBuilder *batch_op_short_ptr = nullptr; - db0::FT_BaseIndex::BatchOperationBuilder *batch_op_long_ptr = nullptr; + TagBaseIndexShortT::BatchOperationBuilder *batch_op_short_ptr = nullptr; + TagBaseIndexLongT::BatchOperationBuilder *batch_op_long_ptr = nullptr; for (std::size_t i = 0; i < nargs; ++i) { auto type_id = LangToolkit::getTypeManager().getTypeId(args[i]); if (isLongTag(type_id, args[i])) { @@ -582,7 +601,7 @@ namespace db0::object_model bool TagIndex::flush() const { - using ShortBatchOperationBulder = db0::FT_BaseIndex::BatchOperationBuilder; + using ShortBatchOperationBulder = TagBaseIndexShortT::BatchOperationBuilder; auto fixture = m_fixture.lock(); std::optional detach_guard; @@ -646,7 +665,8 @@ namespace db0::object_model fixture->detachIterators(); } // the purpose of callback is to incRef objects when a new tag is assigned - std::function add_tag_callback = [&](UniqueAddress obj_addr) { + std::function add_tag_callback = [&](UniqueRef obj_ref) { + auto obj_addr = obj_ref.asUniqueAddress(); auto it = m_object_cache.find(obj_addr); assert(it != m_object_cache.end()); // NOTE: inc-ref as tag @@ -660,7 +680,8 @@ namespace db0::object_model }; auto &batch_op_types = getBatchOperation(m_base_index_short, m_batch_op_types); - std::function remove_tag_callback = [&](UniqueAddress obj_addr) { + std::function remove_tag_callback = [&](UniqueRef obj_ref) { + auto obj_addr = obj_ref.asUniqueAddress(); auto it = m_object_cache.find(obj_addr); // object may not exist if tags are removed post-deletion if (it != m_object_cache.end()) { @@ -672,7 +693,7 @@ namespace db0::object_model // but it will be more efficient to do it here const Class *type_ptr = &LangToolkit::getMemoType(obj_ptr); while (type_ptr) { - batch_op_types->removeTag({ obj_addr, nullptr }, ShortTagT::fromAddress(type_ptr->getAddress())); + batch_op_types->removeTag({ UniqueRef(obj_addr), nullptr }, ShortTagT::fromAddress(type_ptr->getAddress())); type_ptr = type_ptr->getBaseClassPtr(); } } @@ -714,7 +735,7 @@ namespace db0::object_model // NOTE: we check for actual language references (excluding LangCache + TagIndex) if (!LangToolkit::isMemoDropped(obj_ptr) && !LangToolkit::hasMemoAnyRefs(obj_ptr) && !LangToolkit::hasAnyLangRefs(obj_ptr, 2)) { - m_batch_op_types->revert(LangToolkit::getMemoUniqueAddress(obj_ptr)); + m_batch_op_types->revert(UniqueRef(LangToolkit::getMemoUniqueAddress(obj_ptr))); } } // flush all type-tag updates @@ -730,7 +751,7 @@ namespace db0::object_model // erase only entries that have been resolved to a real address. // Pre-cache ref was born in getBatchOperation; drop it here at resolution time. for (auto it = m_active_cache.begin(); it != m_active_cache.end(); ) { - if (it->second.isValid()) { + if (it->second.isResolved()) { m_active_pre_cache.erase(ObjectSharedExtPtr(it->first)); it = m_active_cache.erase(it); } else { @@ -750,7 +771,7 @@ namespace db0::object_model auto object_addr = LangToolkit::getMemoUniqueAddress(item.first); assert(object_addr.isValid()); // initialize active value with the actual object address - item.second = object_addr; + item.second.resolve(object_addr); // add object to cache if (m_object_cache.find(object_addr) == m_object_cache.end()) { m_object_cache.emplace(object_addr, item.first); @@ -766,6 +787,7 @@ namespace db0::object_model db0::FT_ANDIteratorFactory factory; // the negated root-level query components std::vector > neg_iterators; + bool empty_result = false; if (nargs > 0 || type || !native_args.empty()) { // flush pending updates before querying flush(); @@ -800,10 +822,14 @@ namespace db0::object_model if (!result) { // invalidate factory since no matching results exist factory.clear(); + empty_result = true; } } auto query_iterator = factory.release(); + if (empty_result) { + return query_iterator; + } // handle negated query components if (neg_iterators.empty()) { return query_iterator; @@ -1379,11 +1405,11 @@ namespace db0::object_model for (auto it = ForwardIterator(LangToolkit::getIterator(py_arg)), end = ForwardIterator::end(); it != end; ++it) { if (isShortTag(*it)) { ObjectSharedPtr alt_repr = *it; - auto tag_iterator = m_base_index_short.makeIterator(getShortTag(*it, &alt_repr)); + auto tag_iterator = m_base_index_short.makeIterator(getShortTag(*it, &alt_repr)); // use the alternative representation if such exists split_factory.add(std::move(tag_iterator), alt_repr); } else if (isLongTag(*it)) { - auto tag_iterator = m_base_index_long.makeIterator(getLongTag(*it)); + auto tag_iterator = m_base_index_long.makeIterator(getLongTag(*it)); split_factory.add(std::move(tag_iterator), *it); } else { THROWF(db0::InputException) << "Unable to convert to tag: " @@ -1478,15 +1504,15 @@ namespace db0::object_model super_t::detach(); } - db0::FT_BaseIndex &TagIndex::getBaseIndexShort() { + TagIndex::TagBaseIndexShortT &TagIndex::getBaseIndexShort() { return m_base_index_short; } - const db0::FT_BaseIndex &TagIndex::getBaseIndexShort() const { + const TagIndex::TagBaseIndexShortT &TagIndex::getBaseIndexShort() const { return m_base_index_short; } - const db0::FT_BaseIndex &TagIndex::getBaseIndexLong() const { + const TagIndex::TagBaseIndexLongT &TagIndex::getBaseIndexLong() const { return m_base_index_long; } @@ -1530,7 +1556,7 @@ namespace db0::object_model std::unique_ptr TagIndex::makeIterator(ShortTagT tag) const { flush(); - return m_base_index_short.makeIterator(tag); + return m_base_index_short.makeIterator(tag); } std::unique_ptr TagIndex::makeIterator(const std::vector &tag_sequence, @@ -1564,7 +1590,7 @@ namespace db0::object_model keep_alive.push_back(std::move(child_tag_index)); } - return current_tag_index->m_base_index_short.makeIterator( + return current_tag_index->m_base_index_short.makeIterator( tag_sequence.back(), direction, std::vector(tag_sequence) ); } diff --git a/src/dbzero/object_model/tags/TagIndex.hpp b/src/dbzero/object_model/tags/TagIndex.hpp index e0734c01..b5a1e805 100755 --- a/src/dbzero/object_model/tags/TagIndex.hpp +++ b/src/dbzero/object_model/tags/TagIndex.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -57,7 +58,10 @@ DB0_PACKED_END // string tokens and classes are represented as short tags using ShortTagT = db0::TagAddress; using ShortTagIndexMap = db0::VInstanceMap; - using PassiveTag = db0::object_model::PassiveTag; + using TagBaseIndexShortT = db0::FT_BaseIndex; + using TagBaseIndexLongT = db0::FT_BaseIndex; + + using PassiveTag = object_model::PassiveTag; TagIndex(Memspace &memspace, ClassFactory &, EnumFactory &, RC_LimitedStringPool &, VObjectCache &, std::shared_ptr mutation_log); @@ -127,9 +131,9 @@ DB0_PACKED_END void detach() const; - db0::FT_BaseIndex &getBaseIndexShort(); - const db0::FT_BaseIndex &getBaseIndexShort() const; - const db0::FT_BaseIndex &getBaseIndexLong() const; + TagBaseIndexShortT &getBaseIndexShort(); + const TagBaseIndexShortT &getBaseIndexShort() const; + const TagBaseIndexLongT &getBaseIndexLong() const; ShortTagIndexMap *tryGetShortTagIndexMap(); const ShortTagIndexMap *tryGetShortTagIndexMap() const; ShortTagIndexMap &getShortTagIndexMap(); @@ -163,21 +167,55 @@ DB0_PACKED_END private: using TypeId = db0::bindings::TypeId; - using ActiveValueT = typename db0::FT_BaseIndex::ActiveValueT; + using ActiveValueT = typename TagBaseIndexShortT::ActiveValueT; + + struct ActiveAddress + { + UniqueRef m_owning_ref; + UniqueRef m_passive_ref = UniqueRef().asPassive(); + bool m_owning_used = false; + bool m_passive_used = false; + + UniqueRef &getRef(bool passive) + { + if (passive) { + m_passive_used = true; + return m_passive_ref; + } + m_owning_used = true; + return m_owning_ref; + } + + void resolve(UniqueAddress object_addr) + { + if (m_owning_used) { + m_owning_ref = UniqueRef(object_addr); + } + if (m_passive_used) { + m_passive_ref = UniqueRef(object_addr, true); + } + } + + bool isResolved() const + { + return (!m_owning_used || m_owning_ref.isValid()) + && (!m_passive_used || m_passive_ref.isValid()); + } + }; RC_LimitedStringPool &m_string_pool; ClassFactory &m_class_factory; EnumFactory &m_enum_factory; VObjectCache &m_cache; - db0::FT_BaseIndex m_base_index_short; - db0::FT_BaseIndex m_base_index_long; + TagBaseIndexShortT m_base_index_short; + TagBaseIndexLongT m_base_index_long; // For composite tags std::unique_ptr m_short_tag_index_map; // Current batch-operation buffer (may not be initialized) - mutable db0::FT_BaseIndex::BatchOperationBuilder m_batch_op_short; - mutable db0::FT_BaseIndex::BatchOperationBuilder m_batch_op_long; + mutable TagBaseIndexShortT::BatchOperationBuilder m_batch_op_short; + mutable TagBaseIndexLongT::BatchOperationBuilder m_batch_op_long; // batch operation associated with type-tags only (auto-assigned) - mutable db0::FT_BaseIndex::BatchOperationBuilder m_batch_op_types; + mutable TagBaseIndexShortT::BatchOperationBuilder m_batch_op_types; // the set of tags to which the ref-count has been increased when they were first created mutable std::unordered_set m_inc_refed_tags; // A cache of language objects held until flush/close is called @@ -186,7 +224,7 @@ DB0_PACKED_END // NOTE: cache must hold "shared external" references to the objects mutable std::unordered_map m_object_cache; // A cache for incomplete objects (not yet fully initialized) - mutable std::unordered_map m_active_cache; + mutable std::unordered_map m_active_cache; // Additional buffer to preserve / release ownership for active-cache objects mutable std::unordered_set m_active_pre_cache; db0::weak_swine_ptr m_fixture; @@ -198,12 +236,19 @@ DB0_PACKED_END BatchOperationT &getBatchOperation(BaseIndexT &, BatchOperationT &) const; template - BatchOperationT &getBatchOperation(ObjectPtr, BaseIndexT &, BatchOperationT &, ActiveValueT &result) const; + BatchOperationT &getBatchOperation(ObjectPtr, BaseIndexT &, BatchOperationT &, ActiveValueT &result, + bool passive = false) const; + + TagBaseIndexShortT::BatchOperationBuilder &getBatchOperationType(ObjectPtr, ActiveValueT &result) const; + TagBaseIndexShortT::BatchOperationBuilder &getBatchOperationShortTag(ObjectPtr, ActiveValueT &result, + bool passive = false) const; + TagBaseIndexLongT::BatchOperationBuilder &getBatchOperationLongTag(ObjectPtr, ActiveValueT &result, + bool passive = false) const; - db0::FT_BaseIndex::BatchOperationBuilder &getBatchOperationShort(ObjectPtr, + TagBaseIndexShortT::BatchOperationBuilder &getBatchOperationShort(ObjectPtr, ActiveValueT &result, bool is_type) const; - db0::FT_BaseIndex::BatchOperationBuilder &getBatchOperationLong(ObjectPtr, + TagBaseIndexLongT::BatchOperationBuilder &getBatchOperationLong(ObjectPtr, ActiveValueT &result) const; /** @@ -299,7 +344,7 @@ DB0_PACKED_END template BatchOperationT &TagIndex::getBatchOperation(ObjectPtr memo_ptr, BaseIndexT &base_index, - BatchOperationT &batch_op, ActiveValueT &result) const + BatchOperationT &batch_op, ActiveValueT &result, bool passive) const { // prepare the active value only if it's not yet initialized if (!result.first.isValid() && !result.second) { @@ -310,12 +355,12 @@ DB0_PACKED_END if (m_object_cache.find(object_addr) == m_object_cache.end()) { m_object_cache.emplace(object_addr, memo_ptr); } - result = ActiveValueT(object_addr, nullptr); + result = ActiveValueT(UniqueRef(object_addr, passive), nullptr); } else { m_active_pre_cache.insert(memo_ptr); - auto it = m_active_cache.emplace(memo_ptr, UniqueAddress()); + auto it = m_active_cache.emplace(memo_ptr, ActiveAddress()); // use the address placeholder for an active value - result = ActiveValueT(UniqueAddress(), &(it.first->second)); + result = ActiveValueT(UniqueRef(), &(it.first->second.getRef(passive))); } } diff --git a/tests/unit_tests/QuerySerializationTest.cpp b/tests/unit_tests/QuerySerializationTest.cpp index 315e7349..bae8115a 100755 --- a/tests/unit_tests/QuerySerializationTest.cpp +++ b/tests/unit_tests/QuerySerializationTest.cpp @@ -35,7 +35,8 @@ namespace tests return ShortTagT::fromOffset(value); } - void runTestCase(std::function, FT_BaseIndex &)> test) + void runTestCase(std::function, + TagIndex::TagBaseIndexShortT &)> test) { auto fixture = getFixture(); // create with the limit of 8 items per range @@ -74,8 +75,9 @@ namespace tests TEST_F( QuerySerializationTest , testRangeTreeFTSortedIteratorCanBeSerialized ) { - auto test = [](IndexBase &index, std::shared_ptr rt, FT_BaseIndex &ft_index) { - auto ft_query = ft_index.makeIterator(tag(1)); + auto test = [](IndexBase &index, std::shared_ptr rt, + TagIndex::TagBaseIndexShortT &ft_index) { + auto ft_query = ft_index.makeIterator(tag(1)); std::vector values; RT_SortIterator cut(index, rt, std::move(ft_query)); std::vector buf; @@ -87,9 +89,10 @@ namespace tests TEST_F( QuerySerializationTest , testRangeTreeFTSortedIteratorCanBeDeserialized ) { - auto test = [&](IndexBase &index, std::shared_ptr rt, FT_BaseIndex &ft_index) { + auto test = [&](IndexBase &index, std::shared_ptr rt, + TagIndex::TagBaseIndexShortT &ft_index) { std::vector buf; - auto ft_query = ft_index.makeIterator(tag(1)); + auto ft_query = ft_index.makeIterator(tag(1)); std::vector values; RT_SortIterator cut(index, rt, std::move(ft_query)); diff --git a/tests/unit_tests/TagAddressTest.cpp b/tests/unit_tests/TagAddressTest.cpp index 6e5525ff..8b6b0e30 100755 --- a/tests/unit_tests/TagAddressTest.cpp +++ b/tests/unit_tests/TagAddressTest.cpp @@ -13,6 +13,7 @@ namespace tests db0::TagAddress cut; ASSERT_FALSE(cut.isValid()); + ASSERT_FALSE(cut.isPassive()); ASSERT_EQ(cut.getValue(), 0u); ASSERT_EQ(cut.getOffset(), 0u); } @@ -22,42 +23,67 @@ namespace tests auto cut = db0::TagAddress::fromOffset(12345); ASSERT_TRUE(cut.isValid()); + ASSERT_FALSE(cut.isPassive()); ASSERT_EQ(cut.getValue(), 12345u); ASSERT_EQ(cut.getOffset(), 12345u); ASSERT_EQ(cut.getAddress(), db0::Address::fromOffset(12345)); } - TEST( TagAddressTest , testComparisonUsesRawValue ) + TEST( TagAddressTest , testPassiveKeepsRawBitAndStripsLogicalOffset ) { - auto lhs = db0::TagAddress::fromValue(12345); - auto rhs = db0::TagAddress::fromValue(12346); + auto regular = db0::TagAddress::fromOffset(12345); + auto passive = regular.asPassive(); + + ASSERT_TRUE(passive.isValid()); + ASSERT_TRUE(passive.isPassive()); + ASSERT_EQ(passive.getValue(), 12345u | db0::TagAddress::PASSIVE_BIT); + ASSERT_EQ(passive.getOffset(), 12345u); + ASSERT_EQ(passive.getAddress(), db0::Address::fromOffset(12345)); + ASSERT_EQ(passive.asRegular().getValue(), 12345u); + } + + TEST( TagAddressTest , testRawValueCanReopenPassiveAddress ) + { + auto cut = db0::TagAddress::fromValue(12345u | db0::TagAddress::PASSIVE_BIT); + + ASSERT_TRUE(cut.isPassive()); + ASSERT_EQ(cut.getValue(), 12345u | db0::TagAddress::PASSIVE_BIT); + ASSERT_EQ(cut.getOffset(), 12345u); + } + + TEST( TagAddressTest , testRegularAndPassiveCompareAsSameLogicalAddress ) + { + auto regular = db0::TagAddress::fromOffset(12345); + auto passive = regular.asPassive(); - ASSERT_NE(lhs, rhs); - ASSERT_LT(lhs, rhs); + ASSERT_EQ(regular, passive); + ASSERT_FALSE(regular < passive); + ASSERT_FALSE(passive < regular); + ASSERT_LT(regular, db0::TagAddress::fromOffset(12346)); } - TEST( TagAddressTest , testCastsUseRawValue ) + TEST( TagAddressTest , testCastsClearPassiveBit ) { - auto tag = db0::TagAddress::fromValue(12345); + auto passive = db0::TagAddress::fromOffset(12345).asPassive(); - db0::Address address = tag; - std::uint64_t value = tag; + db0::Address address = passive; + std::uint64_t value = passive; ASSERT_EQ(address, db0::Address::fromOffset(12345)); ASSERT_EQ(value, 12345u); } - TEST( TagAddressTest , testHashUsesRawValue ) + TEST( TagAddressTest , testHashUsesLogicalAddress ) { - auto first = db0::TagAddress::fromValue(12345); - auto second = db0::TagAddress::fromValue(12346); + auto regular = db0::TagAddress::fromOffset(12345); + auto passive = regular.asPassive(); std::unordered_set values; - values.insert(first); - values.insert(second); + values.insert(regular); + values.insert(passive); - ASSERT_EQ(values.size(), 2u); - ASSERT_NE(std::hash()(first), std::hash()(second)); + ASSERT_EQ(values.size(), 1u); + ASSERT_EQ(std::hash()(regular), std::hash()(passive)); } TEST( TagAddressTest , testLayoutMatchesUint64 )