diff --git a/luprex/core/Makefile b/luprex/core/Makefile index ba3cc7db..035e3cd9 100644 --- a/luprex/core/Makefile +++ b/luprex/core/Makefile @@ -78,6 +78,7 @@ CORE_OBJ_FILES=\ obj/globaldb.o\ obj/sched.o\ obj/http.o\ + obj/json.o\ obj/table.o\ obj/gui.o\ obj/luasnap.o\ diff --git a/luprex/core/cpp/bytell-hash-map.hpp b/luprex/core/cpp/bytell-hash-map.hpp new file mode 100644 index 00000000..19c4b8be --- /dev/null +++ b/luprex/core/cpp/bytell-hash-map.hpp @@ -0,0 +1,1260 @@ +// Copyright Malte Skarupke 2017. +// Distributed under the Boost Software License, Version 1.0. +// (See http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include "flat-hash-map.hpp" +#include +#include + +namespace ska +{ + +namespace detailv8 +{ +using ska::detailv3::functor_storage; +using ska::detailv3::KeyOrValueHasher; +using ska::detailv3::KeyOrValueEquality; +using ska::detailv3::AssignIfTrue; +using ska::detailv3::HashPolicySelector; + +template +struct sherwood_v8_constants +{ + static constexpr int8_t magic_for_empty = int8_t(0b11111111); + static constexpr int8_t magic_for_reserved = int8_t(0b11111110); + static constexpr int8_t bits_for_direct_hit = int8_t(0b10000000); + static constexpr int8_t magic_for_direct_hit = int8_t(0b00000000); + static constexpr int8_t magic_for_list_entry = int8_t(0b10000000); + + static constexpr int8_t bits_for_distance = int8_t(0b01111111); + inline static int distance_from_metadata(int8_t metadata) + { + return metadata & bits_for_distance; + } + + static constexpr int num_jump_distances = 126; + // jump distances chosen like this: + // 1. pick the first 16 integers to promote staying in the same block + // 2. add the next 66 triangular numbers to get even jumps when + // the hash table is a power of two + // 3. add 44 more triangular numbers at a much steeper growth rate + // to get a sequence that allows large jumps so that a table + // with 10000 sequential numbers doesn't endlessly re-allocate + static constexpr size_t jump_distances[num_jump_distances] + { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + + 21, 28, 36, 45, 55, 66, 78, 91, 105, 120, 136, 153, 171, 190, 210, 231, + 253, 276, 300, 325, 351, 378, 406, 435, 465, 496, 528, 561, 595, 630, + 666, 703, 741, 780, 820, 861, 903, 946, 990, 1035, 1081, 1128, 1176, + 1225, 1275, 1326, 1378, 1431, 1485, 1540, 1596, 1653, 1711, 1770, 1830, + 1891, 1953, 2016, 2080, 2145, 2211, 2278, 2346, 2415, 2485, 2556, + + 3741, 8385, 18915, 42486, 95703, 215496, 485605, 1091503, 2456436, + 5529475, 12437578, 27986421, 62972253, 141700195, 318819126, 717314626, + 1614000520, 3631437253, 8170829695, 18384318876, 41364501751, + 93070021080, 209407709220, 471167588430, 1060127437995, 2385287281530, + 5366895564381, 12075513791265, 27169907873235, 61132301007778, + 137547673121001, 309482258302503, 696335090510256, 1566753939653640, + 3525196427195653, 7931691866727775, 17846306747368716, + 40154190394120111, 90346928493040500, 203280588949935750, + 457381324898247375, 1029107980662394500, 2315492957028380766, + 5209859150892887590, + }; +}; +template +constexpr int8_t sherwood_v8_constants::magic_for_empty; +template +constexpr int8_t sherwood_v8_constants::magic_for_reserved; +template +constexpr int8_t sherwood_v8_constants::bits_for_direct_hit; +template +constexpr int8_t sherwood_v8_constants::magic_for_direct_hit; +template +constexpr int8_t sherwood_v8_constants::magic_for_list_entry; + +template +constexpr int8_t sherwood_v8_constants::bits_for_distance; + +template +constexpr int sherwood_v8_constants::num_jump_distances; +template +constexpr size_t sherwood_v8_constants::jump_distances[num_jump_distances]; + +template +struct sherwood_v8_block +{ + sherwood_v8_block() + { + } + ~sherwood_v8_block() + { + } + int8_t control_bytes[BlockSize]; + union + { + T data[BlockSize]; + }; + + static sherwood_v8_block * empty_block() + { + static std::array empty_bytes = [] + { + std::array result; + result.fill(sherwood_v8_constants<>::magic_for_empty); + return result; + }(); + return reinterpret_cast(&empty_bytes); + } + + int first_empty_index() const + { + for (int i = 0; i < BlockSize; ++i) + { + if (control_bytes[i] == sherwood_v8_constants<>::magic_for_empty) + return i; + } + return -1; + } + + void fill_control_bytes(int8_t value) + { + std::fill(std::begin(control_bytes), std::end(control_bytes), value); + } +}; + +template +class sherwood_v8_table : private ByteAlloc, private Hasher, private Equal +{ + using AllocatorTraits = std::allocator_traits; + using BlockType = sherwood_v8_block; + using BlockPointer = BlockType *; + using BytePointer = typename AllocatorTraits::pointer; + struct convertible_to_iterator; + using Constants = sherwood_v8_constants<>; + +public: + + using value_type = T; + using size_type = size_t; + using difference_type = std::ptrdiff_t; + using hasher = ArgumentHash; + using key_equal = ArgumentEqual; + using allocator_type = ByteAlloc; + using reference = value_type &; + using const_reference = const value_type &; + using pointer = value_type *; + using const_pointer = const value_type *; + + sherwood_v8_table() + { + } + explicit sherwood_v8_table(size_type bucket_count, const ArgumentHash & hash = ArgumentHash(), const ArgumentEqual & equal = ArgumentEqual(), const ArgumentAlloc & alloc = ArgumentAlloc()) + : ByteAlloc(alloc), Hasher(hash), Equal(equal) + { + if (bucket_count) + rehash(bucket_count); + } + sherwood_v8_table(size_type bucket_count, const ArgumentAlloc & alloc) + : sherwood_v8_table(bucket_count, ArgumentHash(), ArgumentEqual(), alloc) + { + } + sherwood_v8_table(size_type bucket_count, const ArgumentHash & hash, const ArgumentAlloc & alloc) + : sherwood_v8_table(bucket_count, hash, ArgumentEqual(), alloc) + { + } + explicit sherwood_v8_table(const ArgumentAlloc & alloc) + : ByteAlloc(alloc) + { + } + template + sherwood_v8_table(It first, It last, size_type bucket_count = 0, const ArgumentHash & hash = ArgumentHash(), const ArgumentEqual & equal = ArgumentEqual(), const ArgumentAlloc & alloc = ArgumentAlloc()) + : sherwood_v8_table(bucket_count, hash, equal, alloc) + { + insert(first, last); + } + template + sherwood_v8_table(It first, It last, size_type bucket_count, const ArgumentAlloc & alloc) + : sherwood_v8_table(first, last, bucket_count, ArgumentHash(), ArgumentEqual(), alloc) + { + } + template + sherwood_v8_table(It first, It last, size_type bucket_count, const ArgumentHash & hash, const ArgumentAlloc & alloc) + : sherwood_v8_table(first, last, bucket_count, hash, ArgumentEqual(), alloc) + { + } + sherwood_v8_table(std::initializer_list il, size_type bucket_count = 0, const ArgumentHash & hash = ArgumentHash(), const ArgumentEqual & equal = ArgumentEqual(), const ArgumentAlloc & alloc = ArgumentAlloc()) + : sherwood_v8_table(bucket_count, hash, equal, alloc) + { + if (bucket_count == 0) + rehash(il.size()); + insert(il.begin(), il.end()); + } + sherwood_v8_table(std::initializer_list il, size_type bucket_count, const ArgumentAlloc & alloc) + : sherwood_v8_table(il, bucket_count, ArgumentHash(), ArgumentEqual(), alloc) + { + } + sherwood_v8_table(std::initializer_list il, size_type bucket_count, const ArgumentHash & hash, const ArgumentAlloc & alloc) + : sherwood_v8_table(il, bucket_count, hash, ArgumentEqual(), alloc) + { + } + sherwood_v8_table(const sherwood_v8_table & other) + : sherwood_v8_table(other, AllocatorTraits::select_on_container_copy_construction(other.get_allocator())) + { + } + sherwood_v8_table(const sherwood_v8_table & other, const ArgumentAlloc & alloc) + : ByteAlloc(alloc), Hasher(other), Equal(other), _max_load_factor(other._max_load_factor) + { + rehash_for_other_container(other); + try + { + insert(other.begin(), other.end()); + } + catch(...) + { + clear(); + deallocate_data(entries, num_slots_minus_one); + throw; + } + } + sherwood_v8_table(sherwood_v8_table && other) noexcept + : ByteAlloc(std::move(other)), Hasher(std::move(other)), Equal(std::move(other)) + , _max_load_factor(other._max_load_factor) + { + swap_pointers(other); + } + sherwood_v8_table(sherwood_v8_table && other, const ArgumentAlloc & alloc) noexcept + : ByteAlloc(alloc), Hasher(std::move(other)), Equal(std::move(other)) + , _max_load_factor(other._max_load_factor) + { + swap_pointers(other); + } + sherwood_v8_table & operator=(const sherwood_v8_table & other) + { + if (this == std::addressof(other)) + return *this; + + clear(); + if (AllocatorTraits::propagate_on_container_copy_assignment::value) + { + if (static_cast(*this) != static_cast(other)) + { + reset_to_empty_state(); + } + AssignIfTrue()(*this, other); + } + _max_load_factor = other._max_load_factor; + static_cast(*this) = other; + static_cast(*this) = other; + rehash_for_other_container(other); + insert(other.begin(), other.end()); + return *this; + } + sherwood_v8_table & operator=(sherwood_v8_table && other) noexcept + { + if (this == std::addressof(other)) + return *this; + else if (AllocatorTraits::propagate_on_container_move_assignment::value) + { + clear(); + reset_to_empty_state(); + AssignIfTrue()(*this, std::move(other)); + swap_pointers(other); + } + else if (static_cast(*this) == static_cast(other)) + { + swap_pointers(other); + } + else + { + clear(); + _max_load_factor = other._max_load_factor; + rehash_for_other_container(other); + for (T & elem : other) + emplace(std::move(elem)); + other.clear(); + } + static_cast(*this) = std::move(other); + static_cast(*this) = std::move(other); + return *this; + } + ~sherwood_v8_table() + { + clear(); + deallocate_data(entries, num_slots_minus_one); + } + + const allocator_type & get_allocator() const + { + return static_cast(*this); + } + const ArgumentEqual & key_eq() const + { + return static_cast(*this); + } + const ArgumentHash & hash_function() const + { + return static_cast(*this); + } + + template + struct templated_iterator + { + private: + friend class sherwood_v8_table; + BlockPointer current = BlockPointer(); + size_t index = 0; + + public: + templated_iterator() + { + } + templated_iterator(BlockPointer entries, size_t index) + : current(entries) + , index(index) + { + } + + using iterator_category = std::forward_iterator_tag; + using value_type = ValueType; + using difference_type = ptrdiff_t; + using pointer = ValueType *; + using reference = ValueType &; + + friend bool operator==(const templated_iterator & lhs, const templated_iterator & rhs) + { + return lhs.index == rhs.index; + } + friend bool operator!=(const templated_iterator & lhs, const templated_iterator & rhs) + { + return !(lhs == rhs); + } + + templated_iterator & operator++() + { + do + { + if (index % BlockSize == 0) + --current; + if (index-- == 0) + break; + } + while(current->control_bytes[index % BlockSize] == Constants::magic_for_empty); + return *this; + } + templated_iterator operator++(int) + { + templated_iterator copy(*this); + ++*this; + return copy; + } + + ValueType & operator*() const + { + return current->data[index % BlockSize]; + } + ValueType * operator->() const + { + return current->data + index % BlockSize; + } + + operator templated_iterator() const + { + return { current, index }; + } + }; + using iterator = templated_iterator; + using const_iterator = templated_iterator; + + iterator begin() + { + size_t num_slots = num_slots_minus_one ? num_slots_minus_one + 1 : 0; + return ++iterator{ entries + num_slots / BlockSize, num_slots }; + } + const_iterator begin() const + { + size_t num_slots = num_slots_minus_one ? num_slots_minus_one + 1 : 0; + return ++iterator{ entries + num_slots / BlockSize, num_slots }; + } + const_iterator cbegin() const + { + return begin(); + } + iterator end() + { + return { entries - 1, std::numeric_limits::max() }; + } + const_iterator end() const + { + return { entries - 1, std::numeric_limits::max() }; + } + const_iterator cend() const + { + return end(); + } + + inline iterator find(const FindKey & key) + { + size_t index = hash_object(key); + size_t num_slots_minus_one = this->num_slots_minus_one; + BlockPointer entries = this->entries; + index = hash_policy.index_for_hash(index, num_slots_minus_one); + bool first = true; + for (;;) + { + size_t block_index = index / BlockSize; + int index_in_block = index % BlockSize; + BlockPointer block = entries + block_index; + int8_t metadata = block->control_bytes[index_in_block]; + if (first) + { + if ((metadata & Constants::bits_for_direct_hit) != Constants::magic_for_direct_hit) + return end(); + first = false; + } + if (compares_equal(key, block->data[index_in_block])) + return { block, index }; + int8_t to_next_index = metadata & Constants::bits_for_distance; + if (to_next_index == 0) + return end(); + index += Constants::jump_distances[to_next_index]; + index = hash_policy.keep_in_range(index, num_slots_minus_one); + } + } + inline const_iterator find(const FindKey & key) const + { + return const_cast(this)->find(key); + } + size_t count(const FindKey & key) const + { + return find(key) == end() ? 0 : 1; + } + std::pair equal_range(const FindKey & key) + { + iterator found = find(key); + if (found == end()) + return { found, found }; + else + return { found, std::next(found) }; + } + std::pair equal_range(const FindKey & key) const + { + const_iterator found = find(key); + if (found == end()) + return { found, found }; + else + return { found, std::next(found) }; + } + + + template + inline std::pair emplace(Key && key, Args &&... args) + { + size_t index = hash_object(key); + size_t num_slots_minus_one = this->num_slots_minus_one; + BlockPointer entries = this->entries; + index = hash_policy.index_for_hash(index, num_slots_minus_one); + bool first = true; + for (;;) + { + size_t block_index = index / BlockSize; + int index_in_block = index % BlockSize; + BlockPointer block = entries + block_index; + int8_t metadata = block->control_bytes[index_in_block]; + if (first) + { + if ((metadata & Constants::bits_for_direct_hit) != Constants::magic_for_direct_hit) + return emplace_direct_hit({ index, block }, std::forward(key), std::forward(args)...); + first = false; + } + if (compares_equal(key, block->data[index_in_block])) + return { { block, index }, false }; + int8_t to_next_index = metadata & Constants::bits_for_distance; + if (to_next_index == 0) + return emplace_new_key({ index, block }, std::forward(key), std::forward(args)...); + index += Constants::jump_distances[to_next_index]; + index = hash_policy.keep_in_range(index, num_slots_minus_one); + } + } + + std::pair insert(const value_type & value) + { + return emplace(value); + } + std::pair insert(value_type && value) + { + return emplace(std::move(value)); + } + template + iterator emplace_hint(const_iterator, Args &&... args) + { + return emplace(std::forward(args)...).first; + } + iterator insert(const_iterator, const value_type & value) + { + return emplace(value).first; + } + iterator insert(const_iterator, value_type && value) + { + return emplace(std::move(value)).first; + } + + template + void insert(It begin, It end) + { + for (; begin != end; ++begin) + { + emplace(*begin); + } + } + void insert(std::initializer_list il) + { + insert(il.begin(), il.end()); + } + + void rehash(size_t num_items) + { + num_items = std::max(num_items, static_cast(std::ceil(num_elements / static_cast(_max_load_factor)))); + if (num_items == 0) + { + reset_to_empty_state(); + return; + } + auto new_prime_index = hash_policy.next_size_over(num_items); + if (num_items == num_slots_minus_one + 1) + return; + size_t num_blocks = num_items / BlockSize; + if (num_items % BlockSize) + ++num_blocks; + size_t memory_requirement = calculate_memory_requirement(num_blocks); + unsigned char * new_memory = &*AllocatorTraits::allocate(*this, memory_requirement); + + BlockPointer new_buckets = reinterpret_cast(new_memory); + + BlockPointer special_end_item = new_buckets + num_blocks; + for (BlockPointer it = new_buckets; it <= special_end_item; ++it) + it->fill_control_bytes(Constants::magic_for_empty); + using std::swap; + swap(entries, new_buckets); + swap(num_slots_minus_one, num_items); + --num_slots_minus_one; + hash_policy.commit(new_prime_index); + num_elements = 0; + if (num_items) + ++num_items; + size_t old_num_blocks = num_items / BlockSize; + if (num_items % BlockSize) + ++old_num_blocks; + for (BlockPointer it = new_buckets, end = new_buckets + old_num_blocks; it != end; ++it) + { + for (int i = 0; i < BlockSize; ++i) + { + int8_t metadata = it->control_bytes[i]; + if (metadata != Constants::magic_for_empty && metadata != Constants::magic_for_reserved) + { + emplace(std::move(it->data[i])); + AllocatorTraits::destroy(*this, it->data + i); + } + } + } + deallocate_data(new_buckets, num_items - 1); + } + + void reserve(size_t num_elements) + { + size_t required_buckets = num_buckets_for_reserve(num_elements); + if (required_buckets > bucket_count()) + rehash(required_buckets); + } + + // the return value is a type that can be converted to an iterator + // the reason for doing this is that it's not free to find the + // iterator pointing at the next element. if you care about the + // next iterator, turn the return value into an iterator + convertible_to_iterator erase(const_iterator to_erase) + { + LinkedListIt current = { to_erase.index, to_erase.current }; + if (current.has_next()) + { + LinkedListIt previous = current; + LinkedListIt next = current.next(*this); + while (next.has_next()) + { + previous = next; + next = next.next(*this); + } + AllocatorTraits::destroy(*this, std::addressof(*current)); + AllocatorTraits::construct(*this, std::addressof(*current), std::move(*next)); + AllocatorTraits::destroy(*this, std::addressof(*next)); + next.set_metadata(Constants::magic_for_empty); + previous.clear_next(); + } + else + { + if (!current.is_direct_hit()) + find_parent_block(current).clear_next(); + AllocatorTraits::destroy(*this, std::addressof(*current)); + current.set_metadata(Constants::magic_for_empty); + } + --num_elements; + return { to_erase.current, to_erase.index }; + } + + iterator erase(const_iterator begin_it, const_iterator end_it) + { + if (begin_it == end_it) + return { begin_it.current, begin_it.index }; + if (std::next(begin_it) == end_it) + return erase(begin_it); + if (begin_it == begin() && end_it == end()) + { + clear(); + return { end_it.current, end_it.index }; + } + std::vector> depth_in_chain; + for (const_iterator it = begin_it; it != end_it; ++it) + { + LinkedListIt list_it(it.index, it.current); + if (list_it.is_direct_hit()) + depth_in_chain.emplace_back(0, list_it); + else + { + LinkedListIt root = find_direct_hit(list_it); + int distance = 1; + for (;;) + { + LinkedListIt next = root.next(*this); + if (next == list_it) + break; + ++distance; + root = next; + } + depth_in_chain.emplace_back(distance, list_it); + } + } + std::sort(depth_in_chain.begin(), depth_in_chain.end(), [](const auto & a, const auto & b) { return a.first < b.first; }); + for (auto it = depth_in_chain.rbegin(), end = depth_in_chain.rend(); it != end; ++it) + { + erase(it->second.it()); + } + + if (begin_it.current->control_bytes[begin_it.index % BlockSize] == Constants::magic_for_empty) + return ++iterator{ begin_it.current, begin_it.index }; + else + return { begin_it.current, begin_it.index }; + } + + size_t erase(const FindKey & key) + { + auto found = find(key); + if (found == end()) + return 0; + else + { + erase(found); + return 1; + } + } + + void clear() + { + if (!num_slots_minus_one) + return; + size_t num_slots = num_slots_minus_one + 1; + size_t num_blocks = num_slots / BlockSize; + if (num_slots % BlockSize) + ++num_blocks; + for (BlockPointer it = entries, end = it + num_blocks; it != end; ++it) + { + for (int i = 0; i < BlockSize; ++i) + { + if (it->control_bytes[i] != Constants::magic_for_empty) + { + AllocatorTraits::destroy(*this, std::addressof(it->data[i])); + it->control_bytes[i] = Constants::magic_for_empty; + } + } + } + num_elements = 0; + } + + void shrink_to_fit() + { + rehash_for_other_container(*this); + } + + void swap(sherwood_v8_table & other) + { + using std::swap; + swap_pointers(other); + swap(static_cast(*this), static_cast(other)); + swap(static_cast(*this), static_cast(other)); + if (AllocatorTraits::propagate_on_container_swap::value) + swap(static_cast(*this), static_cast(other)); + } + + size_t size() const + { + return num_elements; + } + size_t max_size() const + { + return (AllocatorTraits::max_size(*this)) / sizeof(T); + } + size_t bucket_count() const + { + return num_slots_minus_one ? num_slots_minus_one + 1 : 0; + } + size_type max_bucket_count() const + { + return (AllocatorTraits::max_size(*this)) / sizeof(T); + } + size_t bucket(const FindKey & key) const + { + return hash_policy.index_for_hash(hash_object(key), num_slots_minus_one); + } + float load_factor() const + { + return static_cast(num_elements) / (num_slots_minus_one + 1); + } + void max_load_factor(float value) + { + _max_load_factor = value; + } + float max_load_factor() const + { + return _max_load_factor; + } + + bool empty() const + { + return num_elements == 0; + } + +private: + BlockPointer entries = BlockType::empty_block(); + size_t num_slots_minus_one = 0; + typename HashPolicySelector::type hash_policy; + float _max_load_factor = 0.9375f; + size_t num_elements = 0; + + size_t num_buckets_for_reserve(size_t num_elements) const + { + return static_cast(std::ceil(num_elements / static_cast(_max_load_factor))); + } + void rehash_for_other_container(const sherwood_v8_table & other) + { + rehash(std::min(num_buckets_for_reserve(other.size()), other.bucket_count())); + } + bool is_full() const + { + if (!num_slots_minus_one) + return true; + else + return num_elements + 1 > (num_slots_minus_one + 1) * static_cast(_max_load_factor); + } + + void swap_pointers(sherwood_v8_table & other) + { + using std::swap; + swap(hash_policy, other.hash_policy); + swap(entries, other.entries); + swap(num_slots_minus_one, other.num_slots_minus_one); + swap(num_elements, other.num_elements); + swap(_max_load_factor, other._max_load_factor); + } + + struct LinkedListIt + { + size_t index = 0; + BlockPointer block = nullptr; + + LinkedListIt() + { + } + LinkedListIt(size_t index, BlockPointer block) + : index(index), block(block) + { + } + + iterator it() const + { + return { block, index }; + } + int index_in_block() const + { + return index % BlockSize; + } + bool is_direct_hit() const + { + return (metadata() & Constants::bits_for_direct_hit) == Constants::magic_for_direct_hit; + } + bool is_empty() const + { + return metadata() == Constants::magic_for_empty; + } + bool has_next() const + { + return jump_index() != 0; + } + int8_t jump_index() const + { + return Constants::distance_from_metadata(metadata()); + } + int8_t metadata() const + { + return block->control_bytes[index_in_block()]; + } + void set_metadata(int8_t metadata) + { + block->control_bytes[index_in_block()] = metadata; + } + + LinkedListIt next(sherwood_v8_table & table) const + { + int8_t distance = jump_index(); + size_t next_index = table.hash_policy.keep_in_range(index + Constants::jump_distances[distance], table.num_slots_minus_one); + return { next_index, table.entries + next_index / BlockSize }; + } + void set_next(int8_t jump_index) + { + int8_t & metadata = block->control_bytes[index_in_block()]; + metadata = (metadata & ~Constants::bits_for_distance) | jump_index; + } + void clear_next() + { + set_next(0); + } + + value_type & operator*() const + { + return block->data[index_in_block()]; + } + bool operator!() const + { + return !block; + } + explicit operator bool() const + { + return block != nullptr; + } + bool operator==(const LinkedListIt & other) const + { + return index == other.index; + } + bool operator!=(const LinkedListIt & other) const + { + return !(*this == other); + } + }; + + template + SKA_NOINLINE(std::pair) emplace_direct_hit(LinkedListIt block, Args &&... args) + { + using std::swap; + if (is_full()) + { + grow(); + return emplace(std::forward(args)...); + } + if (block.metadata() == Constants::magic_for_empty) + { + AllocatorTraits::construct(*this, std::addressof(*block), std::forward(args)...); + block.set_metadata(Constants::magic_for_direct_hit); + ++num_elements; + return { block.it(), true }; + } + else + { + LinkedListIt parent_block = find_parent_block(block); + std::pair free_block = find_free_index(parent_block); + if (!free_block.first) + { + grow(); + return emplace(std::forward(args)...); + } + value_type new_value(std::forward(args)...); + for (LinkedListIt it = block;;) + { + AllocatorTraits::construct(*this, std::addressof(*free_block.second), std::move(*it)); + AllocatorTraits::destroy(*this, std::addressof(*it)); + parent_block.set_next(free_block.first); + free_block.second.set_metadata(Constants::magic_for_list_entry); + if (!it.has_next()) + { + it.set_metadata(Constants::magic_for_empty); + break; + } + LinkedListIt next = it.next(*this); + it.set_metadata(Constants::magic_for_empty); + block.set_metadata(Constants::magic_for_reserved); + it = next; + parent_block = free_block.second; + free_block = find_free_index(free_block.second); + if (!free_block.first) + { + grow(); + return emplace(std::move(new_value)); + } + } + AllocatorTraits::construct(*this, std::addressof(*block), std::move(new_value)); + block.set_metadata(Constants::magic_for_direct_hit); + ++num_elements; + return { block.it(), true }; + } + } + + template + SKA_NOINLINE(std::pair) emplace_new_key(LinkedListIt parent, Args &&... args) + { + if (is_full()) + { + grow(); + return emplace(std::forward(args)...); + } + std::pair free_block = find_free_index(parent); + if (!free_block.first) + { + grow(); + return emplace(std::forward(args)...); + } + AllocatorTraits::construct(*this, std::addressof(*free_block.second), std::forward(args)...); + free_block.second.set_metadata(Constants::magic_for_list_entry); + parent.set_next(free_block.first); + ++num_elements; + return { free_block.second.it(), true }; + } + + LinkedListIt find_direct_hit(LinkedListIt child) const + { + size_t to_move_hash = hash_object(*child); + size_t to_move_index = hash_policy.index_for_hash(to_move_hash, num_slots_minus_one); + return { to_move_index, entries + to_move_index / BlockSize }; + } + LinkedListIt find_parent_block(LinkedListIt child) + { + LinkedListIt parent_block = find_direct_hit(child); + for (;;) + { + LinkedListIt next = parent_block.next(*this); + if (next == child) + return parent_block; + parent_block = next; + } + } + + std::pair find_free_index(LinkedListIt parent) const + { + for (int8_t jump_index = 1; jump_index < Constants::num_jump_distances; ++jump_index) + { + size_t index = hash_policy.keep_in_range(parent.index + Constants::jump_distances[jump_index], num_slots_minus_one); + BlockPointer block = entries + index / BlockSize; + if (block->control_bytes[index % BlockSize] == Constants::magic_for_empty) + return { jump_index, { index, block } }; + } + return { 0, {} }; + } + + void grow() + { + rehash(std::max(size_t(10), 2 * bucket_count())); + } + + size_t calculate_memory_requirement(size_t num_blocks) + { + size_t memory_required = sizeof(BlockType) * num_blocks; + memory_required += BlockSize; // for metadata of past-the-end pointer + return memory_required; + } + + void deallocate_data(BlockPointer begin, size_t num_slots_minus_one) + { + if (begin == BlockType::empty_block()) + return; + + ++num_slots_minus_one; + size_t num_blocks = num_slots_minus_one / BlockSize; + if (num_slots_minus_one % BlockSize) + ++num_blocks; + size_t memory = calculate_memory_requirement(num_blocks); + unsigned char * as_byte_pointer = reinterpret_cast(begin); + AllocatorTraits::deallocate(*this, typename AllocatorTraits::pointer(as_byte_pointer), memory); + } + + void reset_to_empty_state() + { + deallocate_data(entries, num_slots_minus_one); + entries = BlockType::empty_block(); + num_slots_minus_one = 0; + hash_policy.reset(); + } + + template + size_t hash_object(const U & key) + { + return static_cast(*this)(key); + } + template + size_t hash_object(const U & key) const + { + return static_cast(*this)(key); + } + template + bool compares_equal(const L & lhs, const R & rhs) + { + return static_cast(*this)(lhs, rhs); + } + + struct convertible_to_iterator + { + BlockPointer it; + size_t index; + + operator iterator() + { + if (it->control_bytes[index % BlockSize] == Constants::magic_for_empty) + return ++iterator{it, index}; + else + return { it, index }; + } + operator const_iterator() + { + if (it->control_bytes[index % BlockSize] == Constants::magic_for_empty) + return ++iterator{it, index}; + else + return { it, index }; + } + }; +}; +template +struct AlignmentOr8Bytes +{ + static constexpr size_t value = 8; +}; +template +struct AlignmentOr8Bytes= 1>::type> +{ + static constexpr size_t value = alignof(T); +}; +template +struct CalculateBytellBlockSize; +template +struct CalculateBytellBlockSize +{ + static constexpr size_t this_value = AlignmentOr8Bytes::value; + static constexpr size_t base_value = CalculateBytellBlockSize::value; + static constexpr size_t value = this_value > base_value ? this_value : base_value; +}; +template<> +struct CalculateBytellBlockSize<> +{ + static constexpr size_t value = 8; +}; +} + +template, typename E = std::equal_to, typename A = std::allocator > > +class bytell_hash_map + : public detailv8::sherwood_v8_table + < + std::pair, + K, + H, + detailv8::KeyOrValueHasher, H>, + E, + detailv8::KeyOrValueEquality, E>, + A, + typename std::allocator_traits::template rebind_alloc, + detailv8::CalculateBytellBlockSize::value + > +{ + using Table = detailv8::sherwood_v8_table + < + std::pair, + K, + H, + detailv8::KeyOrValueHasher, H>, + E, + detailv8::KeyOrValueEquality, E>, + A, + typename std::allocator_traits::template rebind_alloc, + detailv8::CalculateBytellBlockSize::value + >; +public: + + using key_type = K; + using mapped_type = V; + + using Table::Table; + bytell_hash_map() + { + } + + inline V & operator[](const K & key) + { + return emplace(key, convertible_to_value()).first->second; + } + inline V & operator[](K && key) + { + return emplace(std::move(key), convertible_to_value()).first->second; + } + V & at(const K & key) + { + auto found = this->find(key); + if (found == this->end()) + throw std::out_of_range("Argument passed to at() was not in the map."); + return found->second; + } + const V & at(const K & key) const + { + auto found = this->find(key); + if (found == this->end()) + throw std::out_of_range("Argument passed to at() was not in the map."); + return found->second; + } + + using Table::emplace; + std::pair emplace() + { + return emplace(key_type(), convertible_to_value()); + } + template + std::pair insert_or_assign(const key_type & key, M && m) + { + auto emplace_result = emplace(key, std::forward(m)); + if (!emplace_result.second) + emplace_result.first->second = std::forward(m); + return emplace_result; + } + template + std::pair insert_or_assign(key_type && key, M && m) + { + auto emplace_result = emplace(std::move(key), std::forward(m)); + if (!emplace_result.second) + emplace_result.first->second = std::forward(m); + return emplace_result; + } + template + typename Table::iterator insert_or_assign(typename Table::const_iterator, const key_type & key, M && m) + { + return insert_or_assign(key, std::forward(m)).first; + } + template + typename Table::iterator insert_or_assign(typename Table::const_iterator, key_type && key, M && m) + { + return insert_or_assign(std::move(key), std::forward(m)).first; + } + + friend bool operator==(const bytell_hash_map & lhs, const bytell_hash_map & rhs) + { + if (lhs.size() != rhs.size()) + return false; + for (const typename Table::value_type & value : lhs) + { + auto found = rhs.find(value.first); + if (found == rhs.end()) + return false; + else if (value.second != found->second) + return false; + } + return true; + } + friend bool operator!=(const bytell_hash_map & lhs, const bytell_hash_map & rhs) + { + return !(lhs == rhs); + } + +private: + struct convertible_to_value + { + operator V() const + { + return V(); + } + }; +}; + +template, typename E = std::equal_to, typename A = std::allocator > +class bytell_hash_set + : public detailv8::sherwood_v8_table + < + T, + T, + H, + detailv8::functor_storage, + E, + detailv8::functor_storage, + A, + typename std::allocator_traits::template rebind_alloc, + detailv8::CalculateBytellBlockSize::value + > +{ + using Table = detailv8::sherwood_v8_table + < + T, + T, + H, + detailv8::functor_storage, + E, + detailv8::functor_storage, + A, + typename std::allocator_traits::template rebind_alloc, + detailv8::CalculateBytellBlockSize::value + >; +public: + + using key_type = T; + + using Table::Table; + bytell_hash_set() + { + } + + template + std::pair emplace(Args &&... args) + { + return Table::emplace(T(std::forward(args)...)); + } + std::pair emplace(const key_type & arg) + { + return Table::emplace(arg); + } + std::pair emplace(key_type & arg) + { + return Table::emplace(arg); + } + std::pair emplace(const key_type && arg) + { + return Table::emplace(std::move(arg)); + } + std::pair emplace(key_type && arg) + { + return Table::emplace(std::move(arg)); + } + + friend bool operator==(const bytell_hash_set & lhs, const bytell_hash_set & rhs) + { + if (lhs.size() != rhs.size()) + return false; + for (const T & value : lhs) + { + if (rhs.find(value) == rhs.end()) + return false; + } + return true; + } + friend bool operator!=(const bytell_hash_set & lhs, const bytell_hash_set & rhs) + { + return !(lhs == rhs); + } +}; + +} // end namespace ska diff --git a/luprex/core/cpp/flat-hash-map.hpp b/luprex/core/cpp/flat-hash-map.hpp new file mode 100644 index 00000000..a8723ee8 --- /dev/null +++ b/luprex/core/cpp/flat-hash-map.hpp @@ -0,0 +1,1496 @@ +// Copyright Malte Skarupke 2017. +// Distributed under the Boost Software License, Version 1.0. +// (See http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _MSC_VER +#define SKA_NOINLINE(...) __declspec(noinline) __VA_ARGS__ +#else +#define SKA_NOINLINE(...) __VA_ARGS__ __attribute__((noinline)) +#endif + +namespace ska +{ +struct prime_number_hash_policy; +struct power_of_two_hash_policy; +struct fibonacci_hash_policy; + +namespace detailv3 +{ +template +struct functor_storage : Functor +{ + functor_storage() = default; + functor_storage(const Functor & functor) + : Functor(functor) + { + } + template + Result operator()(Args &&... args) + { + return static_cast(*this)(std::forward(args)...); + } + template + Result operator()(Args &&... args) const + { + return static_cast(*this)(std::forward(args)...); + } +}; +template +struct functor_storage +{ + typedef Result (*function_ptr)(Args...); + function_ptr function; + functor_storage(function_ptr function) + : function(function) + { + } + Result operator()(Args... args) const + { + return function(std::forward(args)...); + } + operator function_ptr &() + { + return function; + } + operator const function_ptr &() + { + return function; + } +}; +template +struct KeyOrValueHasher : functor_storage +{ + typedef functor_storage hasher_storage; + KeyOrValueHasher() = default; + KeyOrValueHasher(const hasher & hash) + : hasher_storage(hash) + { + } + size_t operator()(const key_type & key) + { + return static_cast(*this)(key); + } + size_t operator()(const key_type & key) const + { + return static_cast(*this)(key); + } + size_t operator()(const value_type & value) + { + return static_cast(*this)(value.first); + } + size_t operator()(const value_type & value) const + { + return static_cast(*this)(value.first); + } + template + size_t operator()(const std::pair & value) + { + return static_cast(*this)(value.first); + } + template + size_t operator()(const std::pair & value) const + { + return static_cast(*this)(value.first); + } +}; +template +struct KeyOrValueEquality : functor_storage +{ + typedef functor_storage equality_storage; + KeyOrValueEquality() = default; + KeyOrValueEquality(const key_equal & equality) + : equality_storage(equality) + { + } + bool operator()(const key_type & lhs, const key_type & rhs) + { + return static_cast(*this)(lhs, rhs); + } + bool operator()(const key_type & lhs, const value_type & rhs) + { + return static_cast(*this)(lhs, rhs.first); + } + bool operator()(const value_type & lhs, const key_type & rhs) + { + return static_cast(*this)(lhs.first, rhs); + } + bool operator()(const value_type & lhs, const value_type & rhs) + { + return static_cast(*this)(lhs.first, rhs.first); + } + template + bool operator()(const key_type & lhs, const std::pair & rhs) + { + return static_cast(*this)(lhs, rhs.first); + } + template + bool operator()(const std::pair & lhs, const key_type & rhs) + { + return static_cast(*this)(lhs.first, rhs); + } + template + bool operator()(const value_type & lhs, const std::pair & rhs) + { + return static_cast(*this)(lhs.first, rhs.first); + } + template + bool operator()(const std::pair & lhs, const value_type & rhs) + { + return static_cast(*this)(lhs.first, rhs.first); + } + template + bool operator()(const std::pair & lhs, const std::pair & rhs) + { + return static_cast(*this)(lhs.first, rhs.first); + } +}; +static constexpr int8_t min_lookups = 4; +template +struct sherwood_v3_entry +{ + sherwood_v3_entry() + { + } + sherwood_v3_entry(int8_t distance_from_desired) + : distance_from_desired(distance_from_desired) + { + } + ~sherwood_v3_entry() + { + } + static sherwood_v3_entry * empty_default_table() + { + static sherwood_v3_entry result[min_lookups] = { {}, {}, {}, {special_end_value} }; + return result; + } + + bool has_value() const + { + return distance_from_desired >= 0; + } + bool is_empty() const + { + return distance_from_desired < 0; + } + bool is_at_desired_position() const + { + return distance_from_desired <= 0; + } + template + void emplace(int8_t distance, Args &&... args) + { + new (std::addressof(value)) T(std::forward(args)...); + distance_from_desired = distance; + } + + void destroy_value() + { + value.~T(); + distance_from_desired = -1; + } + + int8_t distance_from_desired = -1; + static constexpr int8_t special_end_value = 0; + union { T value; }; +}; + +inline int8_t log2(size_t value) +{ + static constexpr int8_t table[64] = + { + 63, 0, 58, 1, 59, 47, 53, 2, + 60, 39, 48, 27, 54, 33, 42, 3, + 61, 51, 37, 40, 49, 18, 28, 20, + 55, 30, 34, 11, 43, 14, 22, 4, + 62, 57, 46, 52, 38, 26, 32, 41, + 50, 36, 17, 19, 29, 10, 13, 21, + 56, 45, 25, 31, 35, 16, 9, 12, + 44, 24, 15, 8, 23, 7, 6, 5 + }; + value |= value >> 1; + value |= value >> 2; + value |= value >> 4; + value |= value >> 8; + value |= value >> 16; + value |= value >> 32; + return table[((value - (value >> 1)) * 0x07EDD5E59A4E28C2) >> 58]; +} + +template +struct AssignIfTrue +{ + void operator()(T & lhs, const T & rhs) + { + lhs = rhs; + } + void operator()(T & lhs, T && rhs) + { + lhs = std::move(rhs); + } +}; +template +struct AssignIfTrue +{ + void operator()(T &, const T &) + { + } + void operator()(T &, T &&) + { + } +}; + +inline size_t next_power_of_two(size_t i) +{ + --i; + i |= i >> 1; + i |= i >> 2; + i |= i >> 4; + i |= i >> 8; + i |= i >> 16; + i |= i >> 32; + ++i; + return i; +} + +template using void_t = void; + +template +struct HashPolicySelector +{ + typedef fibonacci_hash_policy type; +}; +template +struct HashPolicySelector> +{ + typedef typename T::hash_policy type; +}; + +template +class sherwood_v3_table : private EntryAlloc, private Hasher, private Equal +{ + using Entry = detailv3::sherwood_v3_entry; + using AllocatorTraits = std::allocator_traits; + using EntryPointer = typename AllocatorTraits::pointer; + struct convertible_to_iterator; + +public: + + using value_type = T; + using size_type = size_t; + using difference_type = std::ptrdiff_t; + using hasher = ArgumentHash; + using key_equal = ArgumentEqual; + using allocator_type = EntryAlloc; + using reference = value_type &; + using const_reference = const value_type &; + using pointer = value_type *; + using const_pointer = const value_type *; + + sherwood_v3_table() + { + } + explicit sherwood_v3_table(size_type bucket_count, const ArgumentHash & hash = ArgumentHash(), const ArgumentEqual & equal = ArgumentEqual(), const ArgumentAlloc & alloc = ArgumentAlloc()) + : EntryAlloc(alloc), Hasher(hash), Equal(equal) + { + rehash(bucket_count); + } + sherwood_v3_table(size_type bucket_count, const ArgumentAlloc & alloc) + : sherwood_v3_table(bucket_count, ArgumentHash(), ArgumentEqual(), alloc) + { + } + sherwood_v3_table(size_type bucket_count, const ArgumentHash & hash, const ArgumentAlloc & alloc) + : sherwood_v3_table(bucket_count, hash, ArgumentEqual(), alloc) + { + } + explicit sherwood_v3_table(const ArgumentAlloc & alloc) + : EntryAlloc(alloc) + { + } + template + sherwood_v3_table(It first, It last, size_type bucket_count = 0, const ArgumentHash & hash = ArgumentHash(), const ArgumentEqual & equal = ArgumentEqual(), const ArgumentAlloc & alloc = ArgumentAlloc()) + : sherwood_v3_table(bucket_count, hash, equal, alloc) + { + insert(first, last); + } + template + sherwood_v3_table(It first, It last, size_type bucket_count, const ArgumentAlloc & alloc) + : sherwood_v3_table(first, last, bucket_count, ArgumentHash(), ArgumentEqual(), alloc) + { + } + template + sherwood_v3_table(It first, It last, size_type bucket_count, const ArgumentHash & hash, const ArgumentAlloc & alloc) + : sherwood_v3_table(first, last, bucket_count, hash, ArgumentEqual(), alloc) + { + } + sherwood_v3_table(std::initializer_list il, size_type bucket_count = 0, const ArgumentHash & hash = ArgumentHash(), const ArgumentEqual & equal = ArgumentEqual(), const ArgumentAlloc & alloc = ArgumentAlloc()) + : sherwood_v3_table(bucket_count, hash, equal, alloc) + { + if (bucket_count == 0) + rehash(il.size()); + insert(il.begin(), il.end()); + } + sherwood_v3_table(std::initializer_list il, size_type bucket_count, const ArgumentAlloc & alloc) + : sherwood_v3_table(il, bucket_count, ArgumentHash(), ArgumentEqual(), alloc) + { + } + sherwood_v3_table(std::initializer_list il, size_type bucket_count, const ArgumentHash & hash, const ArgumentAlloc & alloc) + : sherwood_v3_table(il, bucket_count, hash, ArgumentEqual(), alloc) + { + } + sherwood_v3_table(const sherwood_v3_table & other) + : sherwood_v3_table(other, AllocatorTraits::select_on_container_copy_construction(other.get_allocator())) + { + } + sherwood_v3_table(const sherwood_v3_table & other, const ArgumentAlloc & alloc) + : EntryAlloc(alloc), Hasher(other), Equal(other), _max_load_factor(other._max_load_factor) + { + rehash_for_other_container(other); + try + { + insert(other.begin(), other.end()); + } + catch(...) + { + clear(); + deallocate_data(entries, num_slots_minus_one, max_lookups); + throw; + } + } + sherwood_v3_table(sherwood_v3_table && other) noexcept + : EntryAlloc(std::move(other)), Hasher(std::move(other)), Equal(std::move(other)) + { + swap_pointers(other); + } + sherwood_v3_table(sherwood_v3_table && other, const ArgumentAlloc & alloc) noexcept + : EntryAlloc(alloc), Hasher(std::move(other)), Equal(std::move(other)) + { + swap_pointers(other); + } + sherwood_v3_table & operator=(const sherwood_v3_table & other) + { + if (this == std::addressof(other)) + return *this; + + clear(); + if (AllocatorTraits::propagate_on_container_copy_assignment::value) + { + if (static_cast(*this) != static_cast(other)) + { + reset_to_empty_state(); + } + AssignIfTrue()(*this, other); + } + _max_load_factor = other._max_load_factor; + static_cast(*this) = other; + static_cast(*this) = other; + rehash_for_other_container(other); + insert(other.begin(), other.end()); + return *this; + } + sherwood_v3_table & operator=(sherwood_v3_table && other) noexcept + { + if (this == std::addressof(other)) + return *this; + else if (AllocatorTraits::propagate_on_container_move_assignment::value) + { + clear(); + reset_to_empty_state(); + AssignIfTrue()(*this, std::move(other)); + swap_pointers(other); + } + else if (static_cast(*this) == static_cast(other)) + { + swap_pointers(other); + } + else + { + clear(); + _max_load_factor = other._max_load_factor; + rehash_for_other_container(other); + for (T & elem : other) + emplace(std::move(elem)); + other.clear(); + } + static_cast(*this) = std::move(other); + static_cast(*this) = std::move(other); + return *this; + } + ~sherwood_v3_table() + { + clear(); + deallocate_data(entries, num_slots_minus_one, max_lookups); + } + + const allocator_type & get_allocator() const + { + return static_cast(*this); + } + const ArgumentEqual & key_eq() const + { + return static_cast(*this); + } + const ArgumentHash & hash_function() const + { + return static_cast(*this); + } + + template + struct templated_iterator + { + templated_iterator() = default; + templated_iterator(EntryPointer current) + : current(current) + { + } + EntryPointer current = EntryPointer(); + + using iterator_category = std::forward_iterator_tag; + using value_type = ValueType; + using difference_type = ptrdiff_t; + using pointer = ValueType *; + using reference = ValueType &; + + friend bool operator==(const templated_iterator & lhs, const templated_iterator & rhs) + { + return lhs.current == rhs.current; + } + friend bool operator!=(const templated_iterator & lhs, const templated_iterator & rhs) + { + return !(lhs == rhs); + } + + templated_iterator & operator++() + { + do + { + ++current; + } + while(current->is_empty()); + return *this; + } + templated_iterator operator++(int) + { + templated_iterator copy(*this); + ++*this; + return copy; + } + + ValueType & operator*() const + { + return current->value; + } + ValueType * operator->() const + { + return std::addressof(current->value); + } + + operator templated_iterator() const + { + return { current }; + } + }; + using iterator = templated_iterator; + using const_iterator = templated_iterator; + + iterator begin() + { + for (EntryPointer it = entries;; ++it) + { + if (it->has_value()) + return { it }; + } + } + const_iterator begin() const + { + for (EntryPointer it = entries;; ++it) + { + if (it->has_value()) + return { it }; + } + } + const_iterator cbegin() const + { + return begin(); + } + iterator end() + { + return { entries + static_cast(num_slots_minus_one + max_lookups) }; + } + const_iterator end() const + { + return { entries + static_cast(num_slots_minus_one + max_lookups) }; + } + const_iterator cend() const + { + return end(); + } + + iterator find(const FindKey & key) + { + size_t index = hash_policy.index_for_hash(hash_object(key), num_slots_minus_one); + EntryPointer it = entries + ptrdiff_t(index); + for (int8_t distance = 0; it->distance_from_desired >= distance; ++distance, ++it) + { + if (compares_equal(key, it->value)) + return { it }; + } + return end(); + } + const_iterator find(const FindKey & key) const + { + return const_cast(this)->find(key); + } + size_t count(const FindKey & key) const + { + return find(key) == end() ? 0 : 1; + } + std::pair equal_range(const FindKey & key) + { + iterator found = find(key); + if (found == end()) + return { found, found }; + else + return { found, std::next(found) }; + } + std::pair equal_range(const FindKey & key) const + { + const_iterator found = find(key); + if (found == end()) + return { found, found }; + else + return { found, std::next(found) }; + } + + template + std::pair emplace(Key && key, Args &&... args) + { + size_t index = hash_policy.index_for_hash(hash_object(key), num_slots_minus_one); + EntryPointer current_entry = entries + ptrdiff_t(index); + int8_t distance_from_desired = 0; + for (; current_entry->distance_from_desired >= distance_from_desired; ++current_entry, ++distance_from_desired) + { + if (compares_equal(key, current_entry->value)) + return { { current_entry }, false }; + } + return emplace_new_key(distance_from_desired, current_entry, std::forward(key), std::forward(args)...); + } + + std::pair insert(const value_type & value) + { + return emplace(value); + } + std::pair insert(value_type && value) + { + return emplace(std::move(value)); + } + template + iterator emplace_hint(const_iterator, Args &&... args) + { + return emplace(std::forward(args)...).first; + } + iterator insert(const_iterator, const value_type & value) + { + return emplace(value).first; + } + iterator insert(const_iterator, value_type && value) + { + return emplace(std::move(value)).first; + } + + template + void insert(It begin, It end) + { + for (; begin != end; ++begin) + { + emplace(*begin); + } + } + void insert(std::initializer_list il) + { + insert(il.begin(), il.end()); + } + + void rehash(size_t num_buckets) + { + num_buckets = std::max(num_buckets, static_cast(std::ceil(num_elements / static_cast(_max_load_factor)))); + if (num_buckets == 0) + { + reset_to_empty_state(); + return; + } + auto new_prime_index = hash_policy.next_size_over(num_buckets); + if (num_buckets == bucket_count()) + return; + int8_t new_max_lookups = compute_max_lookups(num_buckets); + EntryPointer new_buckets(AllocatorTraits::allocate(*this, num_buckets + new_max_lookups)); + EntryPointer special_end_item = new_buckets + static_cast(num_buckets + new_max_lookups - 1); + for (EntryPointer it = new_buckets; it != special_end_item; ++it) + it->distance_from_desired = -1; + special_end_item->distance_from_desired = Entry::special_end_value; + std::swap(entries, new_buckets); + std::swap(num_slots_minus_one, num_buckets); + --num_slots_minus_one; + hash_policy.commit(new_prime_index); + int8_t old_max_lookups = max_lookups; + max_lookups = new_max_lookups; + num_elements = 0; + for (EntryPointer it = new_buckets, end = it + static_cast(num_buckets + old_max_lookups); it != end; ++it) + { + if (it->has_value()) + { + emplace(std::move(it->value)); + it->destroy_value(); + } + } + deallocate_data(new_buckets, num_buckets, old_max_lookups); + } + + void reserve(size_t num_elements) + { + size_t required_buckets = num_buckets_for_reserve(num_elements); + if (required_buckets > bucket_count()) + rehash(required_buckets); + } + + // the return value is a type that can be converted to an iterator + // the reason for doing this is that it's not free to find the + // iterator pointing at the next element. if you care about the + // next iterator, turn the return value into an iterator + convertible_to_iterator erase(const_iterator to_erase) + { + EntryPointer current = to_erase.current; + current->destroy_value(); + --num_elements; + for (EntryPointer next = current + ptrdiff_t(1); !next->is_at_desired_position(); ++current, ++next) + { + current->emplace(next->distance_from_desired - 1, std::move(next->value)); + next->destroy_value(); + } + return { to_erase.current }; + } + + iterator erase(const_iterator begin_it, const_iterator end_it) + { + if (begin_it == end_it) + return { begin_it.current }; + for (EntryPointer it = begin_it.current, end = end_it.current; it != end; ++it) + { + if (it->has_value()) + { + it->destroy_value(); + --num_elements; + } + } + if (end_it == this->end()) + return this->end(); + ptrdiff_t num_to_move = std::min(static_cast(end_it.current->distance_from_desired), end_it.current - begin_it.current); + EntryPointer to_return = end_it.current - num_to_move; + for (EntryPointer it = end_it.current; !it->is_at_desired_position();) + { + EntryPointer target = it - num_to_move; + target->emplace(it->distance_from_desired - num_to_move, std::move(it->value)); + it->destroy_value(); + ++it; + num_to_move = std::min(static_cast(it->distance_from_desired), num_to_move); + } + return { to_return }; + } + + size_t erase(const FindKey & key) + { + auto found = find(key); + if (found == end()) + return 0; + else + { + erase(found); + return 1; + } + } + + void clear() + { + for (EntryPointer it = entries, end = it + static_cast(num_slots_minus_one + max_lookups); it != end; ++it) + { + if (it->has_value()) + it->destroy_value(); + } + num_elements = 0; + } + + void shrink_to_fit() + { + rehash_for_other_container(*this); + } + + void swap(sherwood_v3_table & other) + { + using std::swap; + swap_pointers(other); + swap(static_cast(*this), static_cast(other)); + swap(static_cast(*this), static_cast(other)); + if (AllocatorTraits::propagate_on_container_swap::value) + swap(static_cast(*this), static_cast(other)); + } + + size_t size() const + { + return num_elements; + } + size_t max_size() const + { + return (AllocatorTraits::max_size(*this)) / sizeof(Entry); + } + size_t bucket_count() const + { + return num_slots_minus_one ? num_slots_minus_one + 1 : 0; + } + size_type max_bucket_count() const + { + return (AllocatorTraits::max_size(*this) - min_lookups) / sizeof(Entry); + } + size_t bucket(const FindKey & key) const + { + return hash_policy.index_for_hash(hash_object(key), num_slots_minus_one); + } + float load_factor() const + { + size_t buckets = bucket_count(); + if (buckets) + return static_cast(num_elements) / bucket_count(); + else + return 0; + } + void max_load_factor(float value) + { + _max_load_factor = value; + } + float max_load_factor() const + { + return _max_load_factor; + } + + bool empty() const + { + return num_elements == 0; + } + +private: + EntryPointer entries = Entry::empty_default_table(); + size_t num_slots_minus_one = 0; + typename HashPolicySelector::type hash_policy; + int8_t max_lookups = detailv3::min_lookups - 1; + float _max_load_factor = 0.5f; + size_t num_elements = 0; + + static int8_t compute_max_lookups(size_t num_buckets) + { + int8_t desired = detailv3::log2(num_buckets); + return std::max(detailv3::min_lookups, desired); + } + + size_t num_buckets_for_reserve(size_t num_elements) const + { + return static_cast(std::ceil(num_elements / std::min(0.5, static_cast(_max_load_factor)))); + } + void rehash_for_other_container(const sherwood_v3_table & other) + { + rehash(std::min(num_buckets_for_reserve(other.size()), other.bucket_count())); + } + + void swap_pointers(sherwood_v3_table & other) + { + using std::swap; + swap(hash_policy, other.hash_policy); + swap(entries, other.entries); + swap(num_slots_minus_one, other.num_slots_minus_one); + swap(num_elements, other.num_elements); + swap(max_lookups, other.max_lookups); + swap(_max_load_factor, other._max_load_factor); + } + + template + SKA_NOINLINE(std::pair) emplace_new_key(int8_t distance_from_desired, EntryPointer current_entry, Key && key, Args &&... args) + { + using std::swap; + if (num_slots_minus_one == 0 || distance_from_desired == max_lookups || num_elements + 1 > (num_slots_minus_one + 1) * static_cast(_max_load_factor)) + { + grow(); + return emplace(std::forward(key), std::forward(args)...); + } + else if (current_entry->is_empty()) + { + current_entry->emplace(distance_from_desired, std::forward(key), std::forward(args)...); + ++num_elements; + return { { current_entry }, true }; + } + value_type to_insert(std::forward(key), std::forward(args)...); + swap(distance_from_desired, current_entry->distance_from_desired); + swap(to_insert, current_entry->value); + iterator result = { current_entry }; + for (++distance_from_desired, ++current_entry;; ++current_entry) + { + if (current_entry->is_empty()) + { + current_entry->emplace(distance_from_desired, std::move(to_insert)); + ++num_elements; + return { result, true }; + } + else if (current_entry->distance_from_desired < distance_from_desired) + { + swap(distance_from_desired, current_entry->distance_from_desired); + swap(to_insert, current_entry->value); + ++distance_from_desired; + } + else + { + ++distance_from_desired; + if (distance_from_desired == max_lookups) + { + swap(to_insert, result.current->value); + grow(); + return emplace(std::move(to_insert)); + } + } + } + } + + void grow() + { + rehash(std::max(size_t(4), 2 * bucket_count())); + } + + void deallocate_data(EntryPointer begin, size_t num_slots_minus_one, int8_t max_lookups) + { + if (begin != Entry::empty_default_table()) + { + AllocatorTraits::deallocate(*this, begin, num_slots_minus_one + max_lookups + 1); + } + } + + void reset_to_empty_state() + { + deallocate_data(entries, num_slots_minus_one, max_lookups); + entries = Entry::empty_default_table(); + num_slots_minus_one = 0; + hash_policy.reset(); + max_lookups = detailv3::min_lookups - 1; + } + + template + size_t hash_object(const U & key) + { + return static_cast(*this)(key); + } + template + size_t hash_object(const U & key) const + { + return static_cast(*this)(key); + } + template + bool compares_equal(const L & lhs, const R & rhs) + { + return static_cast(*this)(lhs, rhs); + } + + struct convertible_to_iterator + { + EntryPointer it; + + operator iterator() + { + if (it->has_value()) + return { it }; + else + return ++iterator{it}; + } + operator const_iterator() + { + if (it->has_value()) + return { it }; + else + return ++const_iterator{it}; + } + }; + +}; +} + +struct prime_number_hash_policy +{ + static size_t mod0(size_t) { return 0llu; } + static size_t mod2(size_t hash) { return hash % 2llu; } + static size_t mod3(size_t hash) { return hash % 3llu; } + static size_t mod5(size_t hash) { return hash % 5llu; } + static size_t mod7(size_t hash) { return hash % 7llu; } + static size_t mod11(size_t hash) { return hash % 11llu; } + static size_t mod13(size_t hash) { return hash % 13llu; } + static size_t mod17(size_t hash) { return hash % 17llu; } + static size_t mod23(size_t hash) { return hash % 23llu; } + static size_t mod29(size_t hash) { return hash % 29llu; } + static size_t mod37(size_t hash) { return hash % 37llu; } + static size_t mod47(size_t hash) { return hash % 47llu; } + static size_t mod59(size_t hash) { return hash % 59llu; } + static size_t mod73(size_t hash) { return hash % 73llu; } + static size_t mod97(size_t hash) { return hash % 97llu; } + static size_t mod127(size_t hash) { return hash % 127llu; } + static size_t mod151(size_t hash) { return hash % 151llu; } + static size_t mod197(size_t hash) { return hash % 197llu; } + static size_t mod251(size_t hash) { return hash % 251llu; } + static size_t mod313(size_t hash) { return hash % 313llu; } + static size_t mod397(size_t hash) { return hash % 397llu; } + static size_t mod499(size_t hash) { return hash % 499llu; } + static size_t mod631(size_t hash) { return hash % 631llu; } + static size_t mod797(size_t hash) { return hash % 797llu; } + static size_t mod1009(size_t hash) { return hash % 1009llu; } + static size_t mod1259(size_t hash) { return hash % 1259llu; } + static size_t mod1597(size_t hash) { return hash % 1597llu; } + static size_t mod2011(size_t hash) { return hash % 2011llu; } + static size_t mod2539(size_t hash) { return hash % 2539llu; } + static size_t mod3203(size_t hash) { return hash % 3203llu; } + static size_t mod4027(size_t hash) { return hash % 4027llu; } + static size_t mod5087(size_t hash) { return hash % 5087llu; } + static size_t mod6421(size_t hash) { return hash % 6421llu; } + static size_t mod8089(size_t hash) { return hash % 8089llu; } + static size_t mod10193(size_t hash) { return hash % 10193llu; } + static size_t mod12853(size_t hash) { return hash % 12853llu; } + static size_t mod16193(size_t hash) { return hash % 16193llu; } + static size_t mod20399(size_t hash) { return hash % 20399llu; } + static size_t mod25717(size_t hash) { return hash % 25717llu; } + static size_t mod32401(size_t hash) { return hash % 32401llu; } + static size_t mod40823(size_t hash) { return hash % 40823llu; } + static size_t mod51437(size_t hash) { return hash % 51437llu; } + static size_t mod64811(size_t hash) { return hash % 64811llu; } + static size_t mod81649(size_t hash) { return hash % 81649llu; } + static size_t mod102877(size_t hash) { return hash % 102877llu; } + static size_t mod129607(size_t hash) { return hash % 129607llu; } + static size_t mod163307(size_t hash) { return hash % 163307llu; } + static size_t mod205759(size_t hash) { return hash % 205759llu; } + static size_t mod259229(size_t hash) { return hash % 259229llu; } + static size_t mod326617(size_t hash) { return hash % 326617llu; } + static size_t mod411527(size_t hash) { return hash % 411527llu; } + static size_t mod518509(size_t hash) { return hash % 518509llu; } + static size_t mod653267(size_t hash) { return hash % 653267llu; } + static size_t mod823117(size_t hash) { return hash % 823117llu; } + static size_t mod1037059(size_t hash) { return hash % 1037059llu; } + static size_t mod1306601(size_t hash) { return hash % 1306601llu; } + static size_t mod1646237(size_t hash) { return hash % 1646237llu; } + static size_t mod2074129(size_t hash) { return hash % 2074129llu; } + static size_t mod2613229(size_t hash) { return hash % 2613229llu; } + static size_t mod3292489(size_t hash) { return hash % 3292489llu; } + static size_t mod4148279(size_t hash) { return hash % 4148279llu; } + static size_t mod5226491(size_t hash) { return hash % 5226491llu; } + static size_t mod6584983(size_t hash) { return hash % 6584983llu; } + static size_t mod8296553(size_t hash) { return hash % 8296553llu; } + static size_t mod10453007(size_t hash) { return hash % 10453007llu; } + static size_t mod13169977(size_t hash) { return hash % 13169977llu; } + static size_t mod16593127(size_t hash) { return hash % 16593127llu; } + static size_t mod20906033(size_t hash) { return hash % 20906033llu; } + static size_t mod26339969(size_t hash) { return hash % 26339969llu; } + static size_t mod33186281(size_t hash) { return hash % 33186281llu; } + static size_t mod41812097(size_t hash) { return hash % 41812097llu; } + static size_t mod52679969(size_t hash) { return hash % 52679969llu; } + static size_t mod66372617(size_t hash) { return hash % 66372617llu; } + static size_t mod83624237(size_t hash) { return hash % 83624237llu; } + static size_t mod105359939(size_t hash) { return hash % 105359939llu; } + static size_t mod132745199(size_t hash) { return hash % 132745199llu; } + static size_t mod167248483(size_t hash) { return hash % 167248483llu; } + static size_t mod210719881(size_t hash) { return hash % 210719881llu; } + static size_t mod265490441(size_t hash) { return hash % 265490441llu; } + static size_t mod334496971(size_t hash) { return hash % 334496971llu; } + static size_t mod421439783(size_t hash) { return hash % 421439783llu; } + static size_t mod530980861(size_t hash) { return hash % 530980861llu; } + static size_t mod668993977(size_t hash) { return hash % 668993977llu; } + static size_t mod842879579(size_t hash) { return hash % 842879579llu; } + static size_t mod1061961721(size_t hash) { return hash % 1061961721llu; } + static size_t mod1337987929(size_t hash) { return hash % 1337987929llu; } + static size_t mod1685759167(size_t hash) { return hash % 1685759167llu; } + static size_t mod2123923447(size_t hash) { return hash % 2123923447llu; } + static size_t mod2675975881(size_t hash) { return hash % 2675975881llu; } + static size_t mod3371518343(size_t hash) { return hash % 3371518343llu; } + static size_t mod4247846927(size_t hash) { return hash % 4247846927llu; } + static size_t mod5351951779(size_t hash) { return hash % 5351951779llu; } + static size_t mod6743036717(size_t hash) { return hash % 6743036717llu; } + static size_t mod8495693897(size_t hash) { return hash % 8495693897llu; } + static size_t mod10703903591(size_t hash) { return hash % 10703903591llu; } + static size_t mod13486073473(size_t hash) { return hash % 13486073473llu; } + static size_t mod16991387857(size_t hash) { return hash % 16991387857llu; } + static size_t mod21407807219(size_t hash) { return hash % 21407807219llu; } + static size_t mod26972146961(size_t hash) { return hash % 26972146961llu; } + static size_t mod33982775741(size_t hash) { return hash % 33982775741llu; } + static size_t mod42815614441(size_t hash) { return hash % 42815614441llu; } + static size_t mod53944293929(size_t hash) { return hash % 53944293929llu; } + static size_t mod67965551447(size_t hash) { return hash % 67965551447llu; } + static size_t mod85631228929(size_t hash) { return hash % 85631228929llu; } + static size_t mod107888587883(size_t hash) { return hash % 107888587883llu; } + static size_t mod135931102921(size_t hash) { return hash % 135931102921llu; } + static size_t mod171262457903(size_t hash) { return hash % 171262457903llu; } + static size_t mod215777175787(size_t hash) { return hash % 215777175787llu; } + static size_t mod271862205833(size_t hash) { return hash % 271862205833llu; } + static size_t mod342524915839(size_t hash) { return hash % 342524915839llu; } + static size_t mod431554351609(size_t hash) { return hash % 431554351609llu; } + static size_t mod543724411781(size_t hash) { return hash % 543724411781llu; } + static size_t mod685049831731(size_t hash) { return hash % 685049831731llu; } + static size_t mod863108703229(size_t hash) { return hash % 863108703229llu; } + static size_t mod1087448823553(size_t hash) { return hash % 1087448823553llu; } + static size_t mod1370099663459(size_t hash) { return hash % 1370099663459llu; } + static size_t mod1726217406467(size_t hash) { return hash % 1726217406467llu; } + static size_t mod2174897647073(size_t hash) { return hash % 2174897647073llu; } + static size_t mod2740199326961(size_t hash) { return hash % 2740199326961llu; } + static size_t mod3452434812973(size_t hash) { return hash % 3452434812973llu; } + static size_t mod4349795294267(size_t hash) { return hash % 4349795294267llu; } + static size_t mod5480398654009(size_t hash) { return hash % 5480398654009llu; } + static size_t mod6904869625999(size_t hash) { return hash % 6904869625999llu; } + static size_t mod8699590588571(size_t hash) { return hash % 8699590588571llu; } + static size_t mod10960797308051(size_t hash) { return hash % 10960797308051llu; } + static size_t mod13809739252051(size_t hash) { return hash % 13809739252051llu; } + static size_t mod17399181177241(size_t hash) { return hash % 17399181177241llu; } + static size_t mod21921594616111(size_t hash) { return hash % 21921594616111llu; } + static size_t mod27619478504183(size_t hash) { return hash % 27619478504183llu; } + static size_t mod34798362354533(size_t hash) { return hash % 34798362354533llu; } + static size_t mod43843189232363(size_t hash) { return hash % 43843189232363llu; } + static size_t mod55238957008387(size_t hash) { return hash % 55238957008387llu; } + static size_t mod69596724709081(size_t hash) { return hash % 69596724709081llu; } + static size_t mod87686378464759(size_t hash) { return hash % 87686378464759llu; } + static size_t mod110477914016779(size_t hash) { return hash % 110477914016779llu; } + static size_t mod139193449418173(size_t hash) { return hash % 139193449418173llu; } + static size_t mod175372756929481(size_t hash) { return hash % 175372756929481llu; } + static size_t mod220955828033581(size_t hash) { return hash % 220955828033581llu; } + static size_t mod278386898836457(size_t hash) { return hash % 278386898836457llu; } + static size_t mod350745513859007(size_t hash) { return hash % 350745513859007llu; } + static size_t mod441911656067171(size_t hash) { return hash % 441911656067171llu; } + static size_t mod556773797672909(size_t hash) { return hash % 556773797672909llu; } + static size_t mod701491027718027(size_t hash) { return hash % 701491027718027llu; } + static size_t mod883823312134381(size_t hash) { return hash % 883823312134381llu; } + static size_t mod1113547595345903(size_t hash) { return hash % 1113547595345903llu; } + static size_t mod1402982055436147(size_t hash) { return hash % 1402982055436147llu; } + static size_t mod1767646624268779(size_t hash) { return hash % 1767646624268779llu; } + static size_t mod2227095190691797(size_t hash) { return hash % 2227095190691797llu; } + static size_t mod2805964110872297(size_t hash) { return hash % 2805964110872297llu; } + static size_t mod3535293248537579(size_t hash) { return hash % 3535293248537579llu; } + static size_t mod4454190381383713(size_t hash) { return hash % 4454190381383713llu; } + static size_t mod5611928221744609(size_t hash) { return hash % 5611928221744609llu; } + static size_t mod7070586497075177(size_t hash) { return hash % 7070586497075177llu; } + static size_t mod8908380762767489(size_t hash) { return hash % 8908380762767489llu; } + static size_t mod11223856443489329(size_t hash) { return hash % 11223856443489329llu; } + static size_t mod14141172994150357(size_t hash) { return hash % 14141172994150357llu; } + static size_t mod17816761525534927(size_t hash) { return hash % 17816761525534927llu; } + static size_t mod22447712886978529(size_t hash) { return hash % 22447712886978529llu; } + static size_t mod28282345988300791(size_t hash) { return hash % 28282345988300791llu; } + static size_t mod35633523051069991(size_t hash) { return hash % 35633523051069991llu; } + static size_t mod44895425773957261(size_t hash) { return hash % 44895425773957261llu; } + static size_t mod56564691976601587(size_t hash) { return hash % 56564691976601587llu; } + static size_t mod71267046102139967(size_t hash) { return hash % 71267046102139967llu; } + static size_t mod89790851547914507(size_t hash) { return hash % 89790851547914507llu; } + static size_t mod113129383953203213(size_t hash) { return hash % 113129383953203213llu; } + static size_t mod142534092204280003(size_t hash) { return hash % 142534092204280003llu; } + static size_t mod179581703095829107(size_t hash) { return hash % 179581703095829107llu; } + static size_t mod226258767906406483(size_t hash) { return hash % 226258767906406483llu; } + static size_t mod285068184408560057(size_t hash) { return hash % 285068184408560057llu; } + static size_t mod359163406191658253(size_t hash) { return hash % 359163406191658253llu; } + static size_t mod452517535812813007(size_t hash) { return hash % 452517535812813007llu; } + static size_t mod570136368817120201(size_t hash) { return hash % 570136368817120201llu; } + static size_t mod718326812383316683(size_t hash) { return hash % 718326812383316683llu; } + static size_t mod905035071625626043(size_t hash) { return hash % 905035071625626043llu; } + static size_t mod1140272737634240411(size_t hash) { return hash % 1140272737634240411llu; } + static size_t mod1436653624766633509(size_t hash) { return hash % 1436653624766633509llu; } + static size_t mod1810070143251252131(size_t hash) { return hash % 1810070143251252131llu; } + static size_t mod2280545475268481167(size_t hash) { return hash % 2280545475268481167llu; } + static size_t mod2873307249533267101(size_t hash) { return hash % 2873307249533267101llu; } + static size_t mod3620140286502504283(size_t hash) { return hash % 3620140286502504283llu; } + static size_t mod4561090950536962147(size_t hash) { return hash % 4561090950536962147llu; } + static size_t mod5746614499066534157(size_t hash) { return hash % 5746614499066534157llu; } + static size_t mod7240280573005008577(size_t hash) { return hash % 7240280573005008577llu; } + static size_t mod9122181901073924329(size_t hash) { return hash % 9122181901073924329llu; } + static size_t mod11493228998133068689(size_t hash) { return hash % 11493228998133068689llu; } + static size_t mod14480561146010017169(size_t hash) { return hash % 14480561146010017169llu; } + static size_t mod18446744073709551557(size_t hash) { return hash % 18446744073709551557llu; } + + using mod_function = size_t (*)(size_t); + + mod_function next_size_over(size_t & size) const + { + // prime numbers generated by the following method: + // 1. start with a prime p = 2 + // 2. go to wolfram alpha and get p = NextPrime(2 * p) + // 3. repeat 2. until you overflow 64 bits + // you now have large gaps which you would hit if somebody called reserve() with an unlucky number. + // 4. to fill the gaps for every prime p go to wolfram alpha and get ClosestPrime(p * 2^(1/3)) and ClosestPrime(p * 2^(2/3)) and put those in the gaps + // 5. get PrevPrime(2^64) and put it at the end + static constexpr const size_t prime_list[] = + { + 2llu, 3llu, 5llu, 7llu, 11llu, 13llu, 17llu, 23llu, 29llu, 37llu, 47llu, + 59llu, 73llu, 97llu, 127llu, 151llu, 197llu, 251llu, 313llu, 397llu, + 499llu, 631llu, 797llu, 1009llu, 1259llu, 1597llu, 2011llu, 2539llu, + 3203llu, 4027llu, 5087llu, 6421llu, 8089llu, 10193llu, 12853llu, 16193llu, + 20399llu, 25717llu, 32401llu, 40823llu, 51437llu, 64811llu, 81649llu, + 102877llu, 129607llu, 163307llu, 205759llu, 259229llu, 326617llu, + 411527llu, 518509llu, 653267llu, 823117llu, 1037059llu, 1306601llu, + 1646237llu, 2074129llu, 2613229llu, 3292489llu, 4148279llu, 5226491llu, + 6584983llu, 8296553llu, 10453007llu, 13169977llu, 16593127llu, 20906033llu, + 26339969llu, 33186281llu, 41812097llu, 52679969llu, 66372617llu, + 83624237llu, 105359939llu, 132745199llu, 167248483llu, 210719881llu, + 265490441llu, 334496971llu, 421439783llu, 530980861llu, 668993977llu, + 842879579llu, 1061961721llu, 1337987929llu, 1685759167llu, 2123923447llu, + 2675975881llu, 3371518343llu, 4247846927llu, 5351951779llu, 6743036717llu, + 8495693897llu, 10703903591llu, 13486073473llu, 16991387857llu, + 21407807219llu, 26972146961llu, 33982775741llu, 42815614441llu, + 53944293929llu, 67965551447llu, 85631228929llu, 107888587883llu, + 135931102921llu, 171262457903llu, 215777175787llu, 271862205833llu, + 342524915839llu, 431554351609llu, 543724411781llu, 685049831731llu, + 863108703229llu, 1087448823553llu, 1370099663459llu, 1726217406467llu, + 2174897647073llu, 2740199326961llu, 3452434812973llu, 4349795294267llu, + 5480398654009llu, 6904869625999llu, 8699590588571llu, 10960797308051llu, + 13809739252051llu, 17399181177241llu, 21921594616111llu, 27619478504183llu, + 34798362354533llu, 43843189232363llu, 55238957008387llu, 69596724709081llu, + 87686378464759llu, 110477914016779llu, 139193449418173llu, + 175372756929481llu, 220955828033581llu, 278386898836457llu, + 350745513859007llu, 441911656067171llu, 556773797672909llu, + 701491027718027llu, 883823312134381llu, 1113547595345903llu, + 1402982055436147llu, 1767646624268779llu, 2227095190691797llu, + 2805964110872297llu, 3535293248537579llu, 4454190381383713llu, + 5611928221744609llu, 7070586497075177llu, 8908380762767489llu, + 11223856443489329llu, 14141172994150357llu, 17816761525534927llu, + 22447712886978529llu, 28282345988300791llu, 35633523051069991llu, + 44895425773957261llu, 56564691976601587llu, 71267046102139967llu, + 89790851547914507llu, 113129383953203213llu, 142534092204280003llu, + 179581703095829107llu, 226258767906406483llu, 285068184408560057llu, + 359163406191658253llu, 452517535812813007llu, 570136368817120201llu, + 718326812383316683llu, 905035071625626043llu, 1140272737634240411llu, + 1436653624766633509llu, 1810070143251252131llu, 2280545475268481167llu, + 2873307249533267101llu, 3620140286502504283llu, 4561090950536962147llu, + 5746614499066534157llu, 7240280573005008577llu, 9122181901073924329llu, + 11493228998133068689llu, 14480561146010017169llu, 18446744073709551557llu + }; + static constexpr size_t (* const mod_functions[])(size_t) = + { + &mod0, &mod2, &mod3, &mod5, &mod7, &mod11, &mod13, &mod17, &mod23, &mod29, &mod37, + &mod47, &mod59, &mod73, &mod97, &mod127, &mod151, &mod197, &mod251, &mod313, &mod397, + &mod499, &mod631, &mod797, &mod1009, &mod1259, &mod1597, &mod2011, &mod2539, &mod3203, + &mod4027, &mod5087, &mod6421, &mod8089, &mod10193, &mod12853, &mod16193, &mod20399, + &mod25717, &mod32401, &mod40823, &mod51437, &mod64811, &mod81649, &mod102877, + &mod129607, &mod163307, &mod205759, &mod259229, &mod326617, &mod411527, &mod518509, + &mod653267, &mod823117, &mod1037059, &mod1306601, &mod1646237, &mod2074129, + &mod2613229, &mod3292489, &mod4148279, &mod5226491, &mod6584983, &mod8296553, + &mod10453007, &mod13169977, &mod16593127, &mod20906033, &mod26339969, &mod33186281, + &mod41812097, &mod52679969, &mod66372617, &mod83624237, &mod105359939, &mod132745199, + &mod167248483, &mod210719881, &mod265490441, &mod334496971, &mod421439783, + &mod530980861, &mod668993977, &mod842879579, &mod1061961721, &mod1337987929, + &mod1685759167, &mod2123923447, &mod2675975881, &mod3371518343, &mod4247846927, + &mod5351951779, &mod6743036717, &mod8495693897, &mod10703903591, &mod13486073473, + &mod16991387857, &mod21407807219, &mod26972146961, &mod33982775741, &mod42815614441, + &mod53944293929, &mod67965551447, &mod85631228929, &mod107888587883, &mod135931102921, + &mod171262457903, &mod215777175787, &mod271862205833, &mod342524915839, + &mod431554351609, &mod543724411781, &mod685049831731, &mod863108703229, + &mod1087448823553, &mod1370099663459, &mod1726217406467, &mod2174897647073, + &mod2740199326961, &mod3452434812973, &mod4349795294267, &mod5480398654009, + &mod6904869625999, &mod8699590588571, &mod10960797308051, &mod13809739252051, + &mod17399181177241, &mod21921594616111, &mod27619478504183, &mod34798362354533, + &mod43843189232363, &mod55238957008387, &mod69596724709081, &mod87686378464759, + &mod110477914016779, &mod139193449418173, &mod175372756929481, &mod220955828033581, + &mod278386898836457, &mod350745513859007, &mod441911656067171, &mod556773797672909, + &mod701491027718027, &mod883823312134381, &mod1113547595345903, &mod1402982055436147, + &mod1767646624268779, &mod2227095190691797, &mod2805964110872297, &mod3535293248537579, + &mod4454190381383713, &mod5611928221744609, &mod7070586497075177, &mod8908380762767489, + &mod11223856443489329, &mod14141172994150357, &mod17816761525534927, + &mod22447712886978529, &mod28282345988300791, &mod35633523051069991, + &mod44895425773957261, &mod56564691976601587, &mod71267046102139967, + &mod89790851547914507, &mod113129383953203213, &mod142534092204280003, + &mod179581703095829107, &mod226258767906406483, &mod285068184408560057, + &mod359163406191658253, &mod452517535812813007, &mod570136368817120201, + &mod718326812383316683, &mod905035071625626043, &mod1140272737634240411, + &mod1436653624766633509, &mod1810070143251252131, &mod2280545475268481167, + &mod2873307249533267101, &mod3620140286502504283, &mod4561090950536962147, + &mod5746614499066534157, &mod7240280573005008577, &mod9122181901073924329, + &mod11493228998133068689, &mod14480561146010017169, &mod18446744073709551557 + }; + const size_t * found = std::lower_bound(std::begin(prime_list), std::end(prime_list) - 1, size); + size = *found; + return mod_functions[1 + found - prime_list]; + } + void commit(mod_function new_mod_function) + { + current_mod_function = new_mod_function; + } + void reset() + { + current_mod_function = &mod0; + } + + size_t index_for_hash(size_t hash, size_t /*num_slots_minus_one*/) const + { + return current_mod_function(hash); + } + size_t keep_in_range(size_t index, size_t num_slots_minus_one) const + { + return index > num_slots_minus_one ? current_mod_function(index) : index; + } + +private: + mod_function current_mod_function = &mod0; +}; + +struct power_of_two_hash_policy +{ + size_t index_for_hash(size_t hash, size_t num_slots_minus_one) const + { + return hash & num_slots_minus_one; + } + size_t keep_in_range(size_t index, size_t num_slots_minus_one) const + { + return index_for_hash(index, num_slots_minus_one); + } + int8_t next_size_over(size_t & size) const + { + size = detailv3::next_power_of_two(size); + return 0; + } + void commit(int8_t) + { + } + void reset() + { + } + +}; + +struct fibonacci_hash_policy +{ + size_t index_for_hash(size_t hash, size_t /*num_slots_minus_one*/) const + { + return (11400714819323198485ull * hash) >> shift; + } + size_t keep_in_range(size_t index, size_t num_slots_minus_one) const + { + return index & num_slots_minus_one; + } + + int8_t next_size_over(size_t & size) const + { + size = std::max(size_t(2), detailv3::next_power_of_two(size)); + return 64 - detailv3::log2(size); + } + void commit(int8_t shift) + { + this->shift = shift; + } + void reset() + { + shift = 63; + } + +private: + int8_t shift = 63; +}; + +template, typename E = std::equal_to, typename A = std::allocator > > +class flat_hash_map + : public detailv3::sherwood_v3_table + < + std::pair, + K, + H, + detailv3::KeyOrValueHasher, H>, + E, + detailv3::KeyOrValueEquality, E>, + A, + typename std::allocator_traits::template rebind_alloc>> + > +{ + using Table = detailv3::sherwood_v3_table + < + std::pair, + K, + H, + detailv3::KeyOrValueHasher, H>, + E, + detailv3::KeyOrValueEquality, E>, + A, + typename std::allocator_traits::template rebind_alloc>> + >; +public: + + using key_type = K; + using mapped_type = V; + + using Table::Table; + flat_hash_map() + { + } + + inline V & operator[](const K & key) + { + return emplace(key, convertible_to_value()).first->second; + } + inline V & operator[](K && key) + { + return emplace(std::move(key), convertible_to_value()).first->second; + } + V & at(const K & key) + { + auto found = this->find(key); + if (found == this->end()) + throw std::out_of_range("Argument passed to at() was not in the map."); + return found->second; + } + const V & at(const K & key) const + { + auto found = this->find(key); + if (found == this->end()) + throw std::out_of_range("Argument passed to at() was not in the map."); + return found->second; + } + + using Table::emplace; + std::pair emplace() + { + return emplace(key_type(), convertible_to_value()); + } + template + std::pair insert_or_assign(const key_type & key, M && m) + { + auto emplace_result = emplace(key, std::forward(m)); + if (!emplace_result.second) + emplace_result.first->second = std::forward(m); + return emplace_result; + } + template + std::pair insert_or_assign(key_type && key, M && m) + { + auto emplace_result = emplace(std::move(key), std::forward(m)); + if (!emplace_result.second) + emplace_result.first->second = std::forward(m); + return emplace_result; + } + template + typename Table::iterator insert_or_assign(typename Table::const_iterator, const key_type & key, M && m) + { + return insert_or_assign(key, std::forward(m)).first; + } + template + typename Table::iterator insert_or_assign(typename Table::const_iterator, key_type && key, M && m) + { + return insert_or_assign(std::move(key), std::forward(m)).first; + } + + friend bool operator==(const flat_hash_map & lhs, const flat_hash_map & rhs) + { + if (lhs.size() != rhs.size()) + return false; + for (const typename Table::value_type & value : lhs) + { + auto found = rhs.find(value.first); + if (found == rhs.end()) + return false; + else if (value.second != found->second) + return false; + } + return true; + } + friend bool operator!=(const flat_hash_map & lhs, const flat_hash_map & rhs) + { + return !(lhs == rhs); + } + +private: + struct convertible_to_value + { + operator V() const + { + return V(); + } + }; +}; + +template, typename E = std::equal_to, typename A = std::allocator > +class flat_hash_set + : public detailv3::sherwood_v3_table + < + T, + T, + H, + detailv3::functor_storage, + E, + detailv3::functor_storage, + A, + typename std::allocator_traits::template rebind_alloc> + > +{ + using Table = detailv3::sherwood_v3_table + < + T, + T, + H, + detailv3::functor_storage, + E, + detailv3::functor_storage, + A, + typename std::allocator_traits::template rebind_alloc> + >; +public: + + using key_type = T; + + using Table::Table; + flat_hash_set() + { + } + + template + std::pair emplace(Args &&... args) + { + return Table::emplace(T(std::forward(args)...)); + } + std::pair emplace(const key_type & arg) + { + return Table::emplace(arg); + } + std::pair emplace(key_type & arg) + { + return Table::emplace(arg); + } + std::pair emplace(const key_type && arg) + { + return Table::emplace(std::move(arg)); + } + std::pair emplace(key_type && arg) + { + return Table::emplace(std::move(arg)); + } + + friend bool operator==(const flat_hash_set & lhs, const flat_hash_set & rhs) + { + if (lhs.size() != rhs.size()) + return false; + for (const T & value : lhs) + { + if (rhs.find(value) == rhs.end()) + return false; + } + return true; + } + friend bool operator!=(const flat_hash_set & lhs, const flat_hash_set & rhs) + { + return !(lhs == rhs); + } +}; + + +template +struct power_of_two_std_hash : std::hash +{ + typedef ska::power_of_two_hash_policy hash_policy; +}; + +} // end namespace ska diff --git a/luprex/core/cpp/http.cpp b/luprex/core/cpp/http.cpp index 8e803e16..0be8eef9 100644 --- a/luprex/core/cpp/http.cpp +++ b/luprex/core/cpp/http.cpp @@ -9,6 +9,7 @@ #include "wrap-string.hpp" #include "util.hpp" #include "luastack.hpp" +#include "json.hpp" #include @@ -319,6 +320,38 @@ static void send_error_response(bool http11, int code, eng::string extrainfo, St } } +// HTTP 1.0: An entity body is included with a request message only when the request method calls for one. +// The presence of an entity body in a request is signaled by the inclusion of a Content-Length header field +// in the request message headers. HTTP/1.0 requests containing an entity body must include a valid +// Content-Length header field. +// +// HTTP 1.1: The presence of a message-body in a request is signaled by the inclusion of a Content-Length or +// Transfer-Encoding header field in the request's message-headers +// +static bool request_contains_content(int content_length, std::string_view transfer_encoding) { + return (content_length >= 0) || (!transfer_encoding.empty()); +} + +// HTTP 1.0: For response messages, whether or not an entity body is included with a message is dependent on +// both the request method and the response code. All responses to the HEAD request method must not include a +// body, even though the presence of entity header fields may lead one to believe they do. All 1xx (informational), +// 204 (no content), and 304 (not modified) responses must not include a body. All other responses must include an +// entity body or a Content-Length header field defined with a value of zero (0). +// +// HTTP 1.1: For response messages, whether or not a message-body is included with a message is dependent on both +// the request method and the response status code (section 6.1.1). All responses to the HEAD request method MUST +// NOT include a message-body, even though the presence of entity- header fields might lead one to believe they +// do. All 1xx (informational), 204 (no content), and 304 (not modified) responses MUST NOT include a message-body. +// All other responses do include a message-body, although it MAY be of zero length. +// +static bool response_contains_content(std::string_view method, int status) { + if (method == "HEAD") return false; + if ((status >= 100) && (status <= 199)) return false; + if (status == 204) return false; + if (status == 304) return false; + return true; +} + // In a properly-formed url, the hostname and path are url encoded. // This parser expects an encoded URL. struct ParsedURL { @@ -451,8 +484,8 @@ void HttpClientRequest::send_internal(StreamBuffer *sb, bool debug_string) const sb->write_bytes("\r\n"); } - // If it's a post request, send the content length and the content type. - if (method_ == "POST") { + // Send the content length and the content type. + if (content_assigned_) { send_content_length_header(true, content_.size(), sb); send_content_type_header(true, mime_type_, sb); } @@ -460,8 +493,8 @@ void HttpClientRequest::send_internal(StreamBuffer *sb, bool debug_string) const // Send the extra linebreak. sb->write_bytes("\r\n"); - // If it's a post request, send the content. - if (method_ == "POST") { + // Send the content. + if (content_assigned_) { sb->write_bytes(content_); } } @@ -681,6 +714,17 @@ void HttpClientRequest::set_content(LuaStack &LS, LuaSlot val) { set_content(LS.ckstring(val)); } +void HttpClientRequest::set_jsonvalue(LuaStack &LS, LuaSlot val) { + eng::string out; + eng::string err = json::encode(LS, val, out, false, HttpParser::MAX_CONTENT_LENGTH); + if (!err.empty()) { + check_fail(util::ss("json encode failure: ", err)); + return; + } + set_content(out); + set_mime_type("application/json"); +} + void HttpClientRequest::set_defaults() { if (method_.empty()) { method_ = "GET"; @@ -715,6 +759,20 @@ void HttpClientRequest::set_config(LuaStack &LS0, LuaSlot tab) { set_mime_type(LS, val); } else if (kstr == "content") { set_content(LS, val); + } else if (kstr == "html") { + set_content(LS, val); + set_mime_type("text/html"); + } else if (kstr == "text") { + set_content(LS, val); + set_mime_type("text/plain"); + } else if (kstr == "json") { + set_content(LS, val); + set_mime_type("application/json"); + } else if (kstr == "bytes") { + set_content(LS, val); + set_mime_type("application/octet-stream"); + } else if (kstr == "jsonvalue") { + set_jsonvalue(LS, val); } else if (kstr == "") { check_fail(util::ss("configuration parameter names must be strings.")); } else { @@ -755,13 +813,6 @@ eng::string HttpClientRequest::check() const { if (mime_type_.empty()) { return "mime type not set for POST request"; } - } else { - if (content_assigned_) { - return "content can only be specified for POST requests"; - } - if (!mime_type_.empty()) { - return "mime type can only be specified for POST requests"; - } } return ""; } @@ -822,20 +873,23 @@ void HttpServerResponse::send_internal(StreamBuffer *sb, bool debug_string) cons return; } + // Determine if we're going to be sending content. + bool contains_content = response_contains_content("GET", status_); + // The status message is the human-readable status code. eng::string statusmsg = status_code_to_string(status_); // Annotate error messages. const eng::string *content = &content_; - if (errstatus() && (mime_type_ == "text/plain") && - !content_.empty() && !contains_newline(content_)) { + if (errstatus() && contains_content && + (mime_type_ == "text/plain") && !contains_newline(content_)) { if (sv::has_prefix(content_, statusmsg)) { statusmsg = content_; - } else { + } else if (!content_.empty()) { statusmsg += ": "; statusmsg += content_; - content = &statusmsg; } + content = &statusmsg; } // Send the status line. @@ -848,7 +902,7 @@ void HttpServerResponse::send_internal(StreamBuffer *sb, bool debug_string) cons } // Send the headers that make sense if we're sending content. - if (content_assigned_) { + if (contains_content) { if (!errstatus()) { send_cache_control_header(http11_, max_age_, sb); } @@ -860,7 +914,7 @@ void HttpServerResponse::send_internal(StreamBuffer *sb, bool debug_string) cons sb->write_bytes("\r\n"); // Send the content. - if (content_assigned_) { + if (contains_content) { sb->write_bytes(*content); } } @@ -960,6 +1014,17 @@ void HttpServerResponse::set_content(LuaStack &LS, LuaSlot val) { set_content(LS.ckstring(val)); } +void HttpServerResponse::set_jsonvalue(LuaStack &LS, LuaSlot val) { + eng::string out; + eng::string err = json::encode(LS, val, out, false, HttpParser::MAX_CONTENT_LENGTH); + if (!err.empty()) { + check_fail(util::ss("json encode failure: ", err)); + return; + } + set_content(out); + set_mime_type("application/json"); +} + void HttpServerResponse::set_config(LuaStack &LS0, LuaSlot tab) { LuaVar key, val; LuaStack LS(LS0.state(), key, val); @@ -987,6 +1052,8 @@ void HttpServerResponse::set_config(LuaStack &LS0, LuaSlot tab) { } else if (kstr == "bytes") { set_content(LS, val); set_mime_type("application/octet-stream"); + } else if (kstr == "jsonvalue") { + set_jsonvalue(LS, val); } else if (kstr == "") { check_fail(util::ss("response configuration parameters must be strings.")); } else { @@ -1010,13 +1077,13 @@ void HttpServerResponse::set_defaults() { status_ = 200; } - // If you're sending an error message along with - // an error status code, then assume the error - // message is text/plain. - if ((status_ >= 400) && (status_ <= 599)) { - if (content_assigned_ && (mime_type_.empty())) { - mime_type_ = "text/plain"; - } + // If the response status indicates that we're sending + // content, and there's no content, generate blank content. + if (response_contains_content("GET", status_) && + (!content_assigned_) && (mime_type_.empty())) { + content_assigned_ = true; + content_ = ""; + mime_type_ = "text/plain"; } } @@ -1030,15 +1097,16 @@ eng::string HttpServerResponse::check() const { return "status code not specified"; } - // If you specify content, you have to specify mime - // type, and vice versa. Also, we need both for a valid response. - if ((status_ == 200) || (content_assigned_) || (!mime_type_.empty())) { - if (!content_assigned_) { - return "content not specified"; - } - if (mime_type_.empty()) { - return "mime type not specified"; - } + // If you assigned a mime type and didn't specify + // content, that's bad. + if ((!content_assigned_) && (!mime_type_.empty())) { + return "mime type specified without content"; + } + + // If you assigned content, but didn't assign + // a mime type, that's bad. + if (content_assigned_ && (mime_type_.empty())) { + return "content specified without mime type"; } return ""; } @@ -1337,6 +1405,17 @@ bool HttpParser::parse_content_chunked(std::string_view &view, bool closed) { } bool HttpParser::parse_content(std::string_view &view, bool closed) { + // If there's no body, just return true. + if (is_request_) { + if (!request_contains_content(content_length_, transfer_encoding_)) { + return true; + } + } else { + if (!response_contains_content(method_, status_)) { + return true; + } + } + // Parse the content. if (transfer_encoding_ == "") { if (!parse_content_basic(view, closed)) { @@ -1386,8 +1465,8 @@ bool HttpParser::parse_content(std::string_view &view, bool closed) { } void HttpParser::store(LuaStack &LS0, LuaSlot tab) const { - LuaVar ptab; - LuaStack LS(LS0.state(), ptab); + LuaVar ptab, djson; + LuaStack LS(LS0.state(), ptab, djson); LS.newtable(tab); if (!is_request_) { @@ -1402,6 +1481,10 @@ void HttpParser::store(LuaStack &LS0, LuaSlot tab) const { if (!mime_type_.empty() || !content_.empty()) { LS.rawset(tab, "mimetype", mime_type_); LS.rawset(tab, "content", content_); + if (mime_type_ == "application/json") { + json::decode(LS, djson, content_); + LS.rawset(tab, "jsonvalue", djson); + } } // We currently don't store the method, because // our server only supports GET. Even if we @@ -1475,9 +1558,10 @@ eng::string HttpParser::debug_string() const { return oss.str(); } -void HttpParser::parse_response(std::string_view view, bool closed) { +void HttpParser::parse_response(std::string_view view, bool closed, std::string_view method) { std::string_view original_view = view; is_request_ = false; + method_ = method; // Parse the status line. if (!parse_status_line(view, closed)) { @@ -1524,10 +1608,8 @@ void HttpParser::parse_request(std::string_view view, bool closed) { } // Process the content, if any. - if (method_ == "POST") { - if (!parse_content(view, closed)) { - return; - } + if (!parse_content(view, closed)) { + return; } // Calculate the comm length. @@ -1594,8 +1676,22 @@ LuaDefine(http_clientrequest, "request", "| path (ie, '/index.html')" "| params (a table of url parameters)" "| verifycertificate (default: true)" + "| content (the body to send to the server, if any)" + "| mimetype (mime type of the body, if any)" + "|" + "|The table can also contain this field, which sets" + "|host, port, path, and params in a single directive." + "|" "| url (ie, 'https://host:port/path.html?a=b&c=d')" "|" + "|The table can also contain these fields, which set" + "|both the content and the mimetype in a single directive:" + "|" + "| html - content as a string (text/html)" + "| text - content as a string (text/plain)" + "| json - content as a string (application/json)" + "| bytes - content as a string (application/octet-stream)" + "|" "|You can specify url components separately (host, port, path," "|and params), or you can specify the entire url as a unit. " "|If you specify components, they must not be url-encoded. " @@ -1674,7 +1770,7 @@ LuaDefine(http_clientresponse, "response", LuaRet tab; LuaStack LS(L, text, tab); HttpParser parser; - parser.parse_response(LS.ckstring(text), true); + parser.parse_response(LS.ckstring(text), true, "GET"); parser.store(LS, tab); return LS.result(); } diff --git a/luprex/core/cpp/http.hpp b/luprex/core/cpp/http.hpp index 628ad771..07d174b0 100644 --- a/luprex/core/cpp/http.hpp +++ b/luprex/core/cpp/http.hpp @@ -97,6 +97,7 @@ public: void set_url(LuaStack &LS, LuaSlot val); void set_mime_type(LuaStack &LS, LuaSlot val); void set_content(LuaStack &LS, LuaSlot val); + void set_jsonvalue(LuaStack &LS, LuaSlot val); // Set default values for method and port. // This must be done after setting regular values. @@ -126,6 +127,10 @@ public: // eng::string check() const; + // Get the method. + // + eng::string method() const { return method_; } + // Put the request into the stream, assuming HTTP/1.1 // void send(StreamBuffer *target) const { send_internal(target, false); } @@ -186,6 +191,7 @@ public: // HttpServerResponse(); + void set_method(const eng::string &method); void set_status(int status); void set_max_age(int max_age); void set_mime_type(const eng::string &mime_type); @@ -195,6 +201,7 @@ public: void set_max_age(LuaStack &LS, LuaSlot val); void set_mime_type(LuaStack &LS, LuaSlot val); void set_content(LuaStack &LS, LuaSlot val); + void set_jsonvalue(LuaStack &LS, LuaSlot val); // Set default values. // @@ -275,7 +282,7 @@ protected: // The protocol: true for HTTP/1.1, false for HTTP/1.0 bool http11_; - // The method: always "GET". (only when parsing requests) + // The method. In a response, this is assigned before parsing. eng::string method_; // The URL path, not url-encoded. (only when parsing requests) @@ -383,7 +390,7 @@ public: // The parser will not try to parse content longer than this. // - const int64_t MAX_CONTENT_LENGTH = 1000000; + static constexpr int64_t MAX_CONTENT_LENGTH = 1000000; // Parse a request or a response. // @@ -391,7 +398,7 @@ public: // construct a new parser. // void parse_request(std::string_view view, bool closed); - void parse_response(std::string_view view, bool closed); + void parse_response(std::string_view view, bool closed, std::string_view method); // Store a status code and an error message, and clear the content. // This is generally used when the client detects an error, @@ -428,6 +435,7 @@ using HttpParserVec = eng::vector; class HttpChannel { public: SharedChannel channel_; + eng::string method_; int64_t parsed_bytes_; bool marked_for_deletion() const { return channel_ == nullptr; } diff --git a/luprex/core/cpp/idalloc.cpp b/luprex/core/cpp/idalloc.cpp index c4a908d4..b77c7baa 100644 --- a/luprex/core/cpp/idalloc.cpp +++ b/luprex/core/cpp/idalloc.cpp @@ -87,12 +87,12 @@ void IdGlobalPool::deserialize(StreamBuffer *sb) { eng::string IdGlobalPool::debug_string() const { eng::ostringstream oss; - oss << "next_batch:" << util::hex64() << next_batch_ << " "; - oss << "next_id:" << util::hex64() << next_id_ << " "; - oss << "next_seqno: " << util::hex64() << next_seqno_ << " "; + oss << "next_batch:" << util::hex64.val(next_batch_) << " "; + oss << "next_id:" << util::hex64.val(next_id_) << " "; + oss << "next_seqno: " << util::hex64.val(next_seqno_) << " "; oss << "salvaged:"; for (const int64_t val : salvaged_) { - oss << " " << util::hex64() << val; + oss << " " << util::hex64.val(val); } return oss.str(); } @@ -253,9 +253,9 @@ eng::string IdPlayerPool::debug_string() const { oss << "cap:" << fifo_capacity_ << " ids:"; for (int i = 0; i < int(ranges_.size()); i++) { if (i > 0) oss << ","; - oss << util::hex64() << ranges_[i]; + oss << util::hex64.val(ranges_[i]); } - oss << " seqno:" << util::hex64() << next_seqno_; + oss << " seqno:" << util::hex64.val(next_seqno_); return oss.str(); } diff --git a/luprex/core/cpp/json.cpp b/luprex/core/cpp/json.cpp new file mode 100644 index 00000000..a2ced793 --- /dev/null +++ b/luprex/core/cpp/json.cpp @@ -0,0 +1,728 @@ +#include "json.hpp" +#include "luastack.hpp" +#include "util.hpp" +#include +#include +#include +#include +#include +#include + + +#define NOINDENT_LEVEL 1000 + +LuaTokenConstant(json_null, "null", ""); +LuaTokenConstant(json_object, "object", ""); +LuaTokenConstant(json_error, "error", ""); + +static void indent(eng::ostringstream &oss, int level) { + if (level < NOINDENT_LEVEL) { + oss << std::endl; + for (int i = 0; i < level; i++) { + oss << " "; + } + } +} + +static bool length_exceeded(eng::ostringstream &oss, int maxlen) { + return oss.tellp() > maxlen; +} + +template +inline void store_error(eng::ostringstream &oss, const ARGS & ... args) { + oss.str(""); + util::send_to_stream(oss, args...); +} + +static void store_length_error(eng::ostringstream &oss, int maxlen) { + store_error(oss, "maximum json length exceeded: ", maxlen); +} + +static bool use_array_representation(lua_State *L) { + int top = lua_gettop(L); + int nfound = 0; + while (true) { + lua_rawgeti(L, top, nfound + 1); + bool null = lua_isnil(L, -1); + lua_settop(L, top); + if (null) break; + nfound += 1; + } + return (nfound == lua_nkeys(L, top)); +} + +static bool encode_key(lua_State *L, eng::ostringstream &oss); +static bool encode_value(lua_State *L, eng::ostringstream &oss, int level, int maxlen); + + +// The goal here is to emit a double in such a way that +// when we read it back in, we get the *exact* same number. +// +// In the worst case, you can accomplish this by using 17 +// digits of precision - that's enough to uniquely identify +// all double values (see the following URL). However, 17 +// digits tends to produce unnecessary repeating decimals. +// So we try 16 digits first, which tends to remove those +// repeating decimals, but sometimes produces losses. +// If that doesn't work, we fall back to 17 digits. +// +// https://randomascii.wordpress.com/2012/03/08/float-precisionfrom-zero-to-100-digits-2/ +// +static void encode_double_lossless(double value, eng::ostringstream &oss) { + char buffer[80]; + sprintf(buffer, "%.16g", value); + if (strtod(buffer, nullptr) != value) { + sprintf(buffer, "%.17g", value); + assert(strtod(buffer, nullptr) == value); + } + oss << buffer; +} + +static bool encode_nil(lua_State *L, eng::ostringstream &oss) { + oss << "null"; + return true; +} + +static bool encode_token(lua_State *L, eng::ostringstream &oss) { + LuaToken token(lua_touserdata(L, -1)); + if (token == LuaToken("jsonnull")) { + oss << "null"; + return true; + } else { + store_error(oss, "cannot encode token: [", token.str(), "]"); + return false; + } +} + +static bool encode_number(lua_State *L, eng::ostringstream &oss) { + lua_Number value = lua_tonumber(L, -1); + if (std::isnan(value) || std::isinf(value)) { + store_error(oss, "cannot encode infinity or NAN"); + return false; + } + int64_t ivalue = int64_t(value); + if (double(ivalue) == value) { + oss << ivalue; + } else { + encode_double_lossless(value, oss); + } + return true; +} + +static bool encode_number_key(lua_State *L, eng::ostringstream &oss) { + lua_Number value = lua_tonumber(L, -1); + int64_t ivalue = int64_t(value); + if (double(ivalue) != value) { + store_error(oss, "cannot encode floating point numbers in table keys"); + return false; + } + if (ivalue >= 0) { + oss << "\"\\uE000+" << ivalue << '"'; + } else { + oss << "\"\\uE000-" << -ivalue << '"'; + } + return true; +} + +static bool encode_boolean(lua_State *L, eng::ostringstream &oss) { + int flag = lua_toboolean(L, -1); + oss << (flag ? "true" : "false"); + return true; +} + +static bool encode_string(lua_State *L, eng::ostringstream &oss) { + size_t len; + const char *s = lua_tolstring(L, -1, &len); + std::string_view str(s, len); + oss << '"'; + if (sv::valid_utf8(str) && !sv::has_prefix(str, "")) { + // Output the string in the straightforward way, + // using traditional json escaping. + for (char c : str) { + switch (c) { + case '\\': oss << "\\\\"; break; + case '"' : oss << "\\\""; break; + case '\b': oss << "\\b"; break; + case '\f': oss << "\\f"; break; + case '\r': oss << "\\r"; break; + case '\n': oss << "\\n"; break; + case '\t': oss << "\\t"; break; + default: { + if (c < 32) { + oss << "\\u" << util::hex16.val(c); + } else { + oss << c; + } + } + } + } + } else { + // Output as a base64-encoded string. + oss << "\\uE000="; + util::base64_encode(str, &oss); + } + oss << '"'; + return true; +} + +static bool encode_array(lua_State *L, eng::ostringstream &oss, int level, int maxlen) { + lua_checkstack(L, 20); + int top = lua_gettop(L); + oss << "["; + level ++; + int i = 1; + while (true) { + lua_rawgeti(L, top, i); + if (lua_isnil(L, -1)) break; + if (i > 1) oss << ","; + indent(oss, level); + bool ok = encode_value(L, oss, level, maxlen); + lua_settop(L, top); + if (!ok) return false; + if (length_exceeded(oss, maxlen)) { + store_length_error(oss, maxlen); + return false; + } + i += 1; + } + lua_settop(L, top); + level --; + indent(oss, level); + oss << "]"; + return true; +} + +static bool encode_object(lua_State *L, eng::ostringstream &oss, int level, int maxlen) { + lua_checkstack(L, 20); + int top = lua_gettop(L); + oss << "{"; + level ++; + lua_pushnil(L); + int i = 1; + while (lua_next(L, top) != 0) { + // Check for [json.object]=true, if so skip. + if (lua_islightuserdata(L, -2) && + lua_isboolean(L, -1) && + (LuaToken(lua_touserdata(L, -2)) == LuaToken("object")) && + (lua_toboolean(L, -1) == 1)) { + lua_pop(L, 1); + continue; + } + + lua_pushvalue(L, -2); + // Stack now has key, value, key + assert(lua_gettop(L) == top + 3); + if (i > 1) oss << ","; + indent(oss, level); + bool ok = encode_key(L, oss); + if (!ok) { + lua_settop(L, top); + return false; + } + if (length_exceeded(oss, maxlen)) { + store_length_error(oss, maxlen); + lua_settop(L, top); + return false; + } + lua_pop(L, 1); + // Stack now has key, value + assert(lua_gettop(L) == top + 2); + oss << ((level < NOINDENT_LEVEL) ? " : " : ":"); + ok = encode_value(L, oss, level, maxlen); + assert(lua_gettop(L) == top + 2); + if (!ok) { + lua_settop(L, top); + return false; + } + if (length_exceeded(oss, maxlen)) { + store_length_error(oss, maxlen); + lua_settop(L, top); + return false; + } + lua_pop(L, 1); + // Stack now just has key. + assert(lua_gettop(L) == top + 1); + i += 1; + } + // Stack should be back to where we started. + assert(lua_gettop(L) == top); + level --; + indent(oss, level); + oss << "}"; + return true; +} + +static bool encode_key(lua_State *L, eng::ostringstream &oss) { + int type = lua_type(L, -1); + switch (type) { + case LUA_TSTRING: return encode_string(L, oss); + case LUA_TNUMBER: return encode_number_key(L, oss); + case LUA_TBOOLEAN: + case LUA_TTABLE: { + store_error(oss, "cannot encode '", lua_typename(L, type), "' in table keys"); + return false; + } + default: { + store_error(oss, "cannot encode '", lua_typename(L, type), "'"); + return false; + } + } +} + +static bool encode_value(lua_State *L, eng::ostringstream &oss, int level, int maxlen) { + int type = lua_type(L, -1); + switch (type) { + case LUA_TNIL: return encode_nil(L, oss); + case LUA_TNUMBER: return encode_number(L, oss); + case LUA_TBOOLEAN: return encode_boolean(L, oss); + case LUA_TSTRING: return encode_string(L, oss); + case LUA_TLIGHTUSERDATA: return encode_token(L, oss); + case LUA_TTABLE: { + if (use_array_representation(L)) { + return encode_array(L, oss, level, maxlen); + } else { + return encode_object(L, oss, level, maxlen); + } + } + default: { + store_error(oss, "cannot encode '", lua_typename(L, type), "'"); + return false; + } + } +} + +static bool decode_value(lua_State *L, std::string_view &v); + +static bool decode_id(lua_State *L, std::string_view &v) { + std::string_view id = sv::read_ascii_identifier(v); + if (id == "null") lua_pushlightuserdata(L, LuaToken("null").voidvalue()); + else if (id == "true") lua_pushboolean(L, 1); + else if (id == "false") lua_pushboolean(L, 0); + else return false; + return true; +} + +static bool decode_number(lua_State *L, std::string_view &v) { + std::string_view n = sv::read_number(v, true, true, true, true); + if (n.empty()) return false; + + // If it's an integer, make sure it fits in a lua double + // losslessly. If it's a double, some loss in precision + // is OK. + if (sv::valid_number(n, true, true, false, false)) { + int64_t i = sv::to_int64(n); + if (!LuaStack::int64_storable(i)) return false; + lua_pushnumber(L, double(i)); + return true; + } else { + double d = sv::to_double(n); + if (std::isnan(d) || std::isinf(d)) return false; + lua_pushnumber(L, d); + return true; + } +} + +static bool decode_base64_string(lua_State *L, std::string_view &v) { + // We've already read the starting quote and the E000 + // escape sequence at this point. + + // Skip the equal sign. + if (!sv::read_prefix(v, "=")) return false; + + // Find the end of the quoted string. + const char *p = v.data(); + const char *l = p + v.size(); + while (true) { + if (p == l) return false; + if (*p < 32) return false; + if (*p == '"') break; + p++; + } + std::string_view b64 = v.substr(0, p - v.data()); + v.remove_prefix(b64.size() + 1); + eng::ostringstream oss; + if (!util::base64_decode(b64, &oss)) return false; + eng::string str = oss.str(); + lua_pushlstring(L, str.c_str(), str.size()); + return true; +} + +static bool decode_int_string(lua_State *L, std::string_view &v) { + // We've already read the starting quote and the E000 + // escape sequence at this point. + + // Parse the number and the closing quote. + std::string_view n = sv::read_number(v, true, true, false, false); + if (n.empty()) return false; + if (!sv::read_prefix(v, "\"")) { + return false; + } + + // Make sure the number fits in a lua double, + // and push it on the stack. + int64_t i = sv::to_int64(n); + if (!LuaStack::int64_storable(i)) { + return false; + } + lua_pushnumber(L, double(i)); + return true; +} + +static bool decode_standard_string(lua_State *L, std::string_view &v) { + // We've already read the starting quote at this point. + eng::ostringstream oss; + while (true) { + // Get the next codepoint. + int32_t c = sv::read_codepoint_utf8(v); + + // If it's a control character or invalid codepoint, reject. + if (c < 32) return false; + + // If it is an unescaped quote, that's end of string. + if (c == '"') break; + + // If it's a backslash, then deal with the escape sequence. + if (c == '\\') { + char next = sv::read_ascii_char(v); + switch (next) { + case '"': oss << '"'; break; + case '\\': oss << '\\'; break; + case '/': oss << '/'; break; + case 'r': oss << '\r'; break; + case 'n': oss <<'\n'; break; + case 'b': oss << '\b'; break; + case 'f': oss << '\f'; break; + case 't': oss << '\t'; break; + case 'u': { + std::string_view hexdigits = sv::read_nbytes(v, 4); + if (hexdigits.size() != 4) return false; + uint64_t codepoint = sv::to_hex64(hexdigits, 0x10000); + if (codepoint >= 0x10000) return false; + if (!util::write_codepoint_utf8(codepoint, &oss)) return false; + break; + } + default: return false; + } + continue; + } + + // Any other codepoint should be echoed into stream. + util::write_codepoint_utf8(c, &oss); + } + eng::string result = oss.str(); + lua_pushlstring(L, result.c_str(), result.size()); + return true; +} + +static bool decode_string(lua_State *L, std::string_view &v) { + if (!sv::read_prefix(v, "\"")) return false; + + // Check for codepoint E000, the escape sequence. + if (sv::read_prefix(v, "") || + sv::read_prefix(v, "\\uE000") || + sv::read_prefix(v, "\\ue000")) { + char c = sv::zfront(v); + if (c == '=') return decode_base64_string(L, v); + else if ((c=='-') || (c=='+')) return decode_int_string(L, v); + else return false; + } else { + return decode_standard_string(L, v); + } +} + +static bool decode_array(lua_State *L, std::string_view &v) { + if (!sv::read_prefix(v, "[")) return false; + lua_newtable(L); + int tabpos = lua_gettop(L); + int next = 1; + while (true) { + v = sv::ltrim(v); + if (sv::zfront(v) == ']') { + v.remove_prefix(1); + return true; + } + if (!decode_value(L, v)) { + return false; + } + v = sv::ltrim(v); + if (sv::zfront(v) == ',') { + v.remove_prefix(1); + } + lua_rawseti(L, tabpos, next++); + } +} + +static bool decode_object(lua_State *L, std::string_view &v) { + if (!sv::read_prefix(v, "{")) return false; + lua_newtable(L); + int tabpos = lua_gettop(L); + while (true) { + v = sv::ltrim(v); + if (sv::zfront(v) == '}') { + v.remove_prefix(1); + return true; + } + if (!decode_string(L, v)) { + return false; + } + v = sv::ltrim(v); + if (!sv::read_prefix(v, ":")) { + return false; + } + if (!decode_value(L, v)) { + return false; + } + v = sv::ltrim(v); + if (sv::zfront(v) == ',') { + v.remove_prefix(1); + } + lua_rawset(L, tabpos); + } +} + +// Decode a single value. +// +// On success, pushes the value on the stack and returns true. +// On failure, pushes NIL on the stack and returns false. +// +static bool decode_value(lua_State *L, std::string_view &v) { + lua_checkstack(L, 20); + int top = lua_gettop(L); + + // Skip blanks. + v = sv::ltrim(v); + + // Try to read something. + char c = sv::zfront(v); + bool result; + if (c == '"') result = decode_string(L, v); + else if (c == '[') result = decode_array(L, v); + else if (c == '{') result = decode_object(L, v); + else if (sv::ascii_isalpha(c)) result = decode_id(L, v); + else result = decode_number(L, v); + + // On failure, the decode routines may leave junk + // on the stack, in which case it's our job to clean up. + if (result == false) { + lua_settop(L, top); + lua_pushnil(L); + } + + // Now there should be exactly one new value on the stack. + assert(lua_gettop(L) == top + 1); + return result; +} + +namespace json { + +eng::string encode(LuaStack &LS, LuaSlot in, eng::string &out, bool indent, int maxlen) { + eng::ostringstream oss; + + // Call the recursive encoder. Clean up any crap on the lua stack afterward. + int top = lua_gettop(LS.state()); + lua_pushvalue(LS.state(), in.index()); + bool ok = encode_value(LS.state(), oss, indent ? 0 : NOINDENT_LEVEL, maxlen); + lua_settop(LS.state(), top); + + // One last check for overruns. + if (ok && length_exceeded(oss, maxlen)) { + store_length_error(oss, maxlen); + ok = false; + } + + // Produce the return value. + if (ok) { + out = oss.str(); + return ""; + } else { + out = ""; + return oss.str(); + } +} + +bool decode(LuaStack &LS, LuaSlot out, std::string_view v) { + lua_State *L = LS.state(); + + // Try to read a single value from the view. + int top = lua_gettop(L); + bool ok = decode_value(L, v); + lua_replace(L, out.index()); + lua_settop(L, top); + if (!ok) { + LS.set(out, LuaToken("error")); + return false; + } + + // Special case: if the top level value is 'null', change + // it to 'nil.' + if (LS.istoken(out)) { + LS.set(out, LuaNil); + } + + // There should be nothing left of the input text. + if (v.size() > 0) { + LS.set(out, LuaToken("error")); + return false; + } + return true; +} + +} // namespace util + + +LuaDefine(json_encode, "data, indent, maxlen", + "|Encode a lua data structure returning a json string." + "|" + "|Data is the value being encoded. Indent is a flag," + "|if it's true, then the json is indented nicely," + "|otherwise, it is packed tightly. Maxlen is the maximum" + "|length in bytes of the encoded json string." + "|" + "|Usually, Lua data translates straightforwardly to json." + "|However, there are a number of special cases to be" + "|aware of:" + "|" + "|- Closures and threads cannot be encoded. These will" + "| cause the encoder to abort." + "|" + "|- The numbers infinity and NAN cannot be encoded." + "| Both of these will cause the encoder to abort." + "|" + "|- You must specify a size-limit to the encoded" + "| string. Exceeding the size limit causes the" + "| encoder to abort." + "|" + "|- Recursive data structures will cause the encoder to" + "| loop infinitely until the size-limit is exceeded," + "| causing the encoder to abort." + "|" + "|- There is no way to represent math.huge or math.nan in" + "| json. Encoding math.nan will cause the encoder to abort," + "| as expected. However, encoding math.huge will emit null," + "| which is probably not what you would expect." + "|" + "|- Lua tables cannot contain 'nil', but json objects and" + "| arrays can contain null. If you want the encoder to" + "| emit a json object or array containing null, you must" + "| use token json.null to represent null." + "|" + "|- Json objects, like lua tables, are key-value stores." + "| However, json objects can only have string keys. Our" + "| encoder uses a workaround to transparently" + "| allow mixing string and integer keys in json tables." + "| See 'encoding difficult data' below." + "|" + "|- Json strings are required to be valid utf-8. Our encoder" + "| uses a workaround to transparently allow the use of" + "| arbitrary 8-bit-clean strings. See 'encoding difficult" + "| data' below." + "|" + "|- Lua tables containing contiguous integer keys from 1-n are" + "| autodetected to be json arrays. Empty tables are also" + "| emitted as json arrays. All other tables are emitted" + "| as json objects." + "|" + "|- You can force a table to be emitted as a json object" + "| by putting the key-value pair table[json.object]=true" + "| into the table. This special key is not emitted, but" + "| it triggers json object mode. This is the only way" + "| to emit an empty json object (a truly empty table is" + "| emitted as a json array.)" + "|" + "|Encoding Difficult Data:" + "|" + "|Normally, json doesn't allow integer table keys, and it" + "|doesn't allow strings that aren't valid utf-8. Our" + "|json encoder and decoder, on the other hand, can" + "|encode and decode integer table keys and 8-bit-clean" + "|strings transparently. This is accomplished without" + "|violating the json specification, by encoding such" + "|values as utf-8 strings:" + "|" + "| '123' (encoded integer 123)" + "| '=aGVsbG8=' (binary string encoded as base64)" + "|" + "|Those encodings start with utf-8 codepoint E000." + "|This codepoint probably shows up in your text editor" + "|as a little rectangle. When the decoder sees codepoint" + "|E000 at the beginning of a string, it automatically" + "|decodes the string back into its original form." + "|" + "|The one price for this behavior is that the encoder" + "|cannot literally emit strings that start" + "|with codepoint E000. If the encoder detects such a" + "|string, it will emit it as a base64-encoded string." + "|This should be uncommon, since codepoint E000 is" + "|reserved." + "|" + "|Note that integers are only encoded when they are" + "|used as table keys. Otherwise, numbers are emitted" + "|straightforwardly." + "|") { + LuaArg data, indent, maxlen; + LuaRet encoded; + LuaStack LS(L, data, indent, maxlen, encoded); + eng::string out; + eng::string error = json::encode(LS, data, out, LS.ckboolean(indent), LS.ckint(maxlen)); + if (!error.empty()) { + luaL_error(L, "%s", error.c_str()); + LS.set(encoded, LuaNil); + return LS.result(); + } else { + LS.set(encoded, out); + return LS.result(); + } +} + +LuaDefine(json_decode, "data", + "|Decode a json expression into a lua data structure." + "|" + "|Data that was generated by our own encoder is almost" + "|8-bit clean. That includes difficult cases, like" + "|binary strings, floating point numbers, and tables" + "|with mixed string and integer keys. The exception" + "|are the kinds of data that can't be encoded at all:" + "|See doc(json.encode) for details about what" + "|can and cannot be encoded." + "|" + "|Some json may contain 'null' inside objects and" + "|arrays. Lua tables can't store nil, so instead, we" + "|store the token json.null. If that's not what you" + "|want, you can use json.stripnulls to strip out" + "|the json.null values from a data structure and" + "|replace them with nil." + "|" + "|") { + LuaArg encoded; + LuaRet data; + LuaStack LS(L, encoded, data); + std::string_view v = LS.ckstringview(encoded); + bool ok = json::decode(LS, data, v); + if (!ok) { + luaL_error(L, "invalid json string."); + } + return LS.result(); +} + +// LuaDefine(base64_encode, "data", "") { +// LuaArg str; +// LuaRet ret; +// LuaStack LS(L, str, ret); +// eng::string cstr = LS.ckstring(str); +// eng::ostringstream oss; +// util::base64_encode(cstr, &oss); +// LS.set(ret, oss.str()); +// return LS.result(); +// } + +// LuaDefine(base64_decode, "data", "") { +// LuaArg str; +// LuaRet ret; +// LuaStack LS(L, str, ret); +// eng::string cstr = LS.ckstring(str); +// eng::ostringstream oss; +// util::base64_decode(cstr, &oss); +// LS.set(ret, oss.str()); +// return LS.result(); +// } + diff --git a/luprex/core/cpp/json.hpp b/luprex/core/cpp/json.hpp new file mode 100644 index 00000000..80d0e3a6 --- /dev/null +++ b/luprex/core/cpp/json.hpp @@ -0,0 +1,34 @@ +// Encode lua data structure into json, and decode them again. +// +// See the doc(http.jsonencode) to read about limitations of the encoder. +// +#ifndef JSON_HPP +#define JSON_HPP + +#include "luastack.hpp" +#include "wrap-string.hpp" +#include + +namespace json { + // Encode json. + // + // See doc(http.jsonencode) for a lot more information. + // + // Returns an error message. If the error message is an + // empty string, then the encoding was successful. + // + eng::string encode(LuaStack &LS, LuaSlot in, eng::string &out, bool indent, int maxlen); + + // Decode json. + // + // See doc(http.jsondecode) for a lot more information. + // + // The only error condition is syntactically invalid json. + // In that case, we return false and set 'out' to the + // token 'error'. + // + bool decode(LuaStack &LS, LuaSlot out, std::string_view in); +} + +#endif // JSON_HPP + diff --git a/luprex/core/cpp/lpxserver.cpp b/luprex/core/cpp/lpxserver.cpp index 20162c9b..27dffcc4 100644 --- a/luprex/core/cpp/lpxserver.cpp +++ b/luprex/core/cpp/lpxserver.cpp @@ -219,6 +219,7 @@ public: HttpChannel &channel = http_client_channels_[request.request_id()]; if (channel.channel_ == nullptr) { channel.channel_ = new_outgoing_channel(request.target()); + channel.method_ = request.method(); channel.parsed_bytes_ = 0; request.send(channel.channel_->out()); } @@ -235,7 +236,7 @@ public: response.fail(503, util::ss("Service Unavailable: ", channel.error())); } else { htchan.parsed_bytes_ = channel.in()->fill(); - response.parse_response(channel.in()->view(), channel.closed()); + response.parse_response(channel.in()->view(), channel.closed(), htchan.method_); } if (response.complete()) { response.set_request_id(pair.first); diff --git a/luprex/core/cpp/luastack.cpp b/luprex/core/cpp/luastack.cpp index 75904bbb..c938fa4b 100644 --- a/luprex/core/cpp/luastack.cpp +++ b/luprex/core/cpp/luastack.cpp @@ -2,6 +2,7 @@ #include #include #include +#include LuaSpecial LuaRegistry(LUA_REGISTRYINDEX); LuaNilMarker LuaNil; @@ -17,6 +18,15 @@ LuaFunctionReg::LuaFunctionReg(const char *n, const char *a, const char *d, bool All = this; } +LuaConstantReg::LuaConstantReg(const char *n, const char *d, LuaToken tokenvalue, lua_Number numbervalue) { + name_ = n; + docs_ = d; + tokenvalue_ = tokenvalue; + numbervalue_ = numbervalue; + next_ = All; + All = this; +} + const LuaFunctionReg *LuaFunctionReg::lookup(lua_CFunction fn) { for (const LuaFunctionReg *r = All; r != 0; r = r->next_) { if (r->func_ == fn) { @@ -27,6 +37,20 @@ const LuaFunctionReg *LuaFunctionReg::lookup(lua_CFunction fn) { } LuaFunctionReg *LuaFunctionReg::All; +LuaConstantReg *LuaConstantReg::All; + + +eng::string LuaToken::str() const { + uint64_t token = (uint64_t)value; + char buffer[9]; + for (int i = 0; i < 8; i++) { + unsigned char c = token; + buffer[7-i] = c; + token >>= 8; + } + buffer[8] = 0; + return eng::string(buffer); +} static int panicf(lua_State *L) { const char *p = lua_tostring(L, -1); @@ -107,11 +131,23 @@ eng::string LuaStack::ckstring(LuaSlot s) const { return eng::string(str, len); } +std::string_view LuaStack::ckstringview(LuaSlot s) const { + luaL_checktype(L_, s, LUA_TSTRING); + size_t len; + const char *str = lua_tolstring(L_, s, &len); + return std::string_view(str, len); +} + lua_State *LuaStack::ckthread(LuaSlot s) const { luaL_checktype(L_, s, LUA_TTHREAD); return lua_tothread(L_, s); } +LuaToken LuaStack::cktoken(LuaSlot s) const { + luaL_checktype(L_, s, LUA_TLIGHTUSERDATA); + return LuaToken(lua_touserdata(L_, s)); +} + void LuaStack::count_slots_finalize(int narg, int nvar, int nret) { narg_ = narg; nret_ = nret; diff --git a/luprex/core/cpp/luastack.hpp b/luprex/core/cpp/luastack.hpp index 2d427984..4c5d852c 100644 --- a/luprex/core/cpp/luastack.hpp +++ b/luprex/core/cpp/luastack.hpp @@ -226,6 +226,40 @@ int LuaTypeTagValue(lua_State *L) { return 0; } #define LUA_TT_GLOBALDB 22 #define LUA_TT_CLASS 23 +// We use lightuserdata to store 'tokens': short +// strings of 8 characters or less. These tokens +// are useful as unique markers. The 8 characters +// are packed into a uint64. + +struct LuaToken { +private: + static constexpr uint64_t literal_to_token(const char *str) { + uint64_t result = 0; + for (int i = 0; i < 8; i++) { + unsigned char c = *str; + result = (result << 8) + c; + if (*str) str++; + } + return result; + } +public: + uint64_t value; + + template + LuaToken(T arg) = delete; + + constexpr LuaToken(const char *str) : value(literal_to_token(str)) {} + LuaToken(uint64_t v) : value(v) {} + LuaToken(void *v) : value((uint64_t)v) {} + LuaToken() : value(0) {} + + bool empty() const { return value == 0; } + bool operator ==(const LuaToken &other) const { return value == other.value; } + void *voidvalue() const { return (void*)value; } + + eng::string str() const; +}; + class LuaStack : public eng::nevernew { private: int narg_; @@ -302,6 +336,7 @@ private: void push_any_value(lua_Integer s) const { lua_pushinteger(L_, s); } void push_any_value(lua_CFunction s) const { lua_pushcfunction(L_, s); } void push_any_value(bool b) const { lua_pushboolean(L_, b ? 1:0); } + void push_any_value(LuaToken token) const { lua_pushlightuserdata(L_, (void*)(token.value)); } // Push multiple values on the stack, in order, by type. template @@ -354,6 +389,7 @@ public: bool isboolean(LuaSlot s) const { return lua_type(L_, s) == LUA_TBOOLEAN; } bool isnil(LuaSlot s) const { return lua_type(L_, s) == LUA_TNIL; } bool iscfunction(LuaSlot s) const { return lua_iscfunction(L_, s) != 0; } + bool istoken(LuaSlot s) const { return lua_islightuserdata(L_, s) != 0; } void checktable(LuaSlot index) const { checktype(index, LUA_TTABLE); } void checkstring(LuaSlot index) const { checktype(index, LUA_TSTRING); } @@ -362,13 +398,16 @@ public: void checkfunction(LuaSlot index) const { checktype(index, LUA_TFUNCTION); } void checkboolean(LuaSlot index) const { checktype(index, LUA_TBOOLEAN); } void checknil(LuaSlot index) const { checktype(index, LUA_TNIL); } + void checktoken(LuaSlot index) const { checktype(index, LUA_TLIGHTUSERDATA); } bool ckboolean(LuaSlot s) const; lua_Integer ckinteger(LuaSlot s) const; int ckint(LuaSlot s) const; lua_Number cknumber(LuaSlot s) const; eng::string ckstring(LuaSlot s) const; + std::string_view ckstringview(LuaSlot s) const; lua_State *ckthread(LuaSlot s) const; + LuaToken cktoken(LuaSlot s) const; void clearmetatable(LuaSlot tab) const; void setmetatable(LuaSlot tab, LuaSlot mt) const; @@ -472,8 +511,29 @@ public: // Lua flagbits manipulation: visited bit. bool getvisited(LuaSlot tab) const; void setvisited(LuaSlot tab, bool visited) const; + + // Return true if the int64 value can be stored as a lua number. + static bool int64_storable(int64_t v) { return (v <= MAXINT) && (v >= -MAXINT); } }; +class LuaConstantReg : public eng::nevernew { +private: + const char *name_; + const char *docs_; + LuaToken tokenvalue_; + lua_Number numbervalue_; + LuaConstantReg *next_; + +public: + static LuaConstantReg *All; + LuaConstantReg(const char *name, const char *docs, LuaToken tokenvalue, lua_Number numbervalue); + + const char *get_name() const { return name_; } + const char *get_docs() const { return docs_; } + LuaToken get_tokenvalue() const { return tokenvalue_; } + lua_Number get_numbervalue() const { return numbervalue_; } + LuaConstantReg *next() const { return next_; } +}; class LuaFunctionReg : public eng::nevernew { private: @@ -498,6 +558,11 @@ public: void set_func(lua_CFunction fn) { func_ = fn; } }; +#define LuaTokenConstant(name, tvalue, docs) \ + LuaConstantReg reg_##name(#name, docs, LuaToken(tvalue), 0); + +#define LuaNumberConstant(name, nvalue, docs) \ + LuaConstantReg reg_##name(#name, docs, LuaToken(), nvalue); #define LuaDefine(name, args, docs) \ int lfn_##name(lua_State *L); \ @@ -518,4 +583,5 @@ public: #define LuaStringify(x) #x #define LuaAssert(L, x) if (!(x)) { luaL_error((L), "Assert failed: %s (file %s line %d)", LuaStringify(x), __FILE__, __LINE__); } #define LuaAssertStrEq(L, x, y) { eng::string _s1_(x); eng::string _s2_(y); if (_s1_ != _s2_) luaL_error((L), "Assert failed: value=%s (file %s line %d)", _s1_.c_str(), __FILE__, __LINE__); } + #endif // LUASTACK_HPP diff --git a/luprex/core/cpp/planemap.cpp b/luprex/core/cpp/planemap.cpp index 24e74160..cfefce3e 100644 --- a/luprex/core/cpp/planemap.cpp +++ b/luprex/core/cpp/planemap.cpp @@ -1,4 +1,106 @@ + +// PlaneTree is an octree-like data structure. +// +// THE COORDINATE SPACE +// +// The tree stores objects which have (x,y,z) coordinates where each of x,y,z is +// an unsigned 16-bit integer. The API uses floating-point coordinates, but +// these are converted to uint16_t for use in the tree. +// +// In an octree, each node has a 2x2x2 array of children (total: 8 children). In +// this tree, however, each node has a 4x4x4 array of children (total: 64 +// children). +// +// The depth of the tree is fixed. It always has one leaf level (level 0), plus +// 8 internal levels (levels 1-8). The root of the tree is level 8. Tangibles +// are always are always stored in the leaf-level. Since there are 8 +// subdivisions, at 4x4x4 per subdivision, that divides the space into 65536 x +// 65536 x 65536. Hence the 16-bit integer coordinates. +// +// Here are the sizes of the levels: +// +// * Level 8 (internal) is 1 cubed. +// * Level 7 (internal) is 4 cubed. +// * Level 6 (internal) is 16 cubed. +// * Level 5 (internal) is 64 cubed. +// * Level 4 (internal) is 256 cubed. +// * Level 3 (internal) is 1024 cubed. +// * Level 2 (internal) is 4096 cubed. +// * Level 1 (internal) is 16385 cubed. +// * Level 0 (leaf) is 65536 cubed. +// +// NODE IDS AND THE NODE TABLES +// +// Every tree node has a unique "NodeID", which consists of it's level and the +// lowest (x,y,z) coordinate in its region. Every tree node is stored in a hash +// table that maps NodeID to the data for that tree node. +// +// Tree nodes don't contain any child pointers or parent pointers. Instead, you +// can calculate the NodeIDs of the parent and children from the NodeID of a +// node. So, you can just look up parents and children in the hash table. Even +// though PlaneTree really is a tree, it doesn't use pointers at all. +// +// There are two separate hash tables - one for storing internal nodes, and one +// for storing leaf nodes. The only data stored in an internal node is a +// bitvector indicating which of its 64 children are nonempty. The only data +// stored in a leaf node is a list of PlaneItems. The list of PlaneItems is +// stored as an intrusive doubly-linked circular list. +// +// THE PLANETREE BOUNDING BOX, OUTLIERS, AND ADAPTIVE RESIZING +// +// The planetree as a whole has a bounding box. For now, the bounding box is +// fixed at (-65536, -65536, -65536) to (65536, 65536, 65536). That makes the +// size of the leaf cells two meters. However, it is my plan to eventually have +// the bounding box selected adaptively. +// +// "Outliers" are objects that are outside the PlaneTree's bounding box. When an +// object is an outlier, its (X,Y,Z) coordinates are clamped to the tree's +// bounding box for purposes of storing it in the tree. This puts all the +// outliers into the cells on the edge of the tree. When we need to scan a +// region that includes areas outside the tree's bounding box, we scan the edge +// cells. +// +// The tree keeps a count of the number of outliers. The intended use for these +// outlier counts is to implement adaptive logic that says "are there too many +// outliers? If so, expand the tree's bounding box." But we haven't written the +// adaptive algorithm yet, so the outlier counts don't serve any purpose yet. +// +// We probably *don't* want to capture 100% of the outliers. It seems likely +// that the programmer will deliberately put a few objects way off in the +// distance. For example, you might imagine somebody creating a "LoginManager" +// object and putting it way off at (1000000, 0, 0) in order to get it out of +// the way. If we were to expand the bounding box to include such objects, we +// would make the tree's bounding box unreasonably large. I have an idea for a +// heuristic: expand the bounding box until it covers 90% of the tangibles, then +// expand it by 25% more, then stop. I have another idea for a different +// heuristic: keep expanding the bounding box as long as making a small +// increment will capture significantly more tangibles, then stop. I don't know +// if either of those heuristics will work well. +// +// Any adaptive algorithm we create must take another factor into account: we +// don't want the cell size to be too small. Overly small cell sizes create lots +// of extra work when an object moves around. So sometimes, it's advantageous +// to deliberately expand the bounding box way beyond what is actually needed to +// capture the outliers, just to make the cell size large enough. Doing this +// isn't wasteful: octrees are very good at handling big empty spaces inside the +// bounding box. As long as the cell size is large enough, but not too large, +// and there aren't too many outliers, everything will work well. Unfortunately, +// I haven't devised a good heuristic to decide when the cell size is "right." +// +// Another thought I have, periodically, is that writing a lot of logic just to +// choose a cell size seems like a lot of unnecessary work: we could just +// configure the cell size for the game in the script. +// +// BYPASSING THE TREE +// +// In many cases, it's possible to jump directly to a leaf node and start +// processing. For example, let's say you want to move a tangible. You know +// its current position, therefore, you know exactly the NodeID of the leaf node +// where it is stored. There is no need to walk the tree to get there: you can +// just access it directly using the hash table. +// + #include "luastack.hpp" #include "util.hpp" #include "planemap.hpp" @@ -6,313 +108,1313 @@ #include #include +using NodeID = uint64_t; +using ChildBits = uint64_t; +using IdVector = util::IdVector; -// Cell X, Y coordinates are packed such that they have 24 bits for X and Y. -// A cell is 10 Meters square. -// Cell ID zero is used to represent an invalid position. -// -#define CELL_LIMIT 0x7FFFFF -#define CELL_SCALE 10.0 +// These need to be static floats to encourage gcc to generate +// efficient code in NodeInfo. +static float k_lo = 0.0f; +static float k_hi = 65535.0f; -// Round a float and return as integer. Clamp result to the specified range. -static int round_and_clamp(float x, int lo, int hi) { - x = round(x); - if (x < lo) return lo; - if (x > hi) return hi; - return int(x); +static constexpr ChildBits child_bit(int i) { + return (uint64_t(1) << i); } -// A cell range is inclusive. -struct CellRange { - int xlo; - int ylo; - int xhi; - int yhi; +static constexpr ChildBits child_bit(int x, int y, int z) { + return child_bit((x << 4) | (y << 2) | z); +} - bool equal(int xl, int yl, int xh, int yh) { - return ((xlo==xl)&&(ylo==yl)&&(xh==xhi)&&(yh==yhi)); +static constexpr uint8_t node_get_level(NodeID node) { + return node >> 48; +} + +static constexpr uint16_t node_get_x(NodeID node) { + return uint16_t(node >> 32); +} + +static constexpr uint16_t node_get_y(NodeID node) { + return uint16_t(node >> 16); +} + +static constexpr uint16_t node_get_z(NodeID node) { + return uint16_t(node >> 0); +} + +static constexpr NodeID node_lxyz(uint8_t level, uint16_t x, uint16_t y, uint16_t z) { + return (NodeID(level) << 48) | (NodeID(x) << 32) | (NodeID(y) << 16) | (NodeID(z) << 0); +} + +static constexpr NodeID node_parent(NodeID node) { + uint8_t level = node_get_level(node); + return ((node & node_lxyz(0, 0xFFFC, 0xFFFC, 0xFFFC)) >> 2) | node_lxyz(level + 1, 0, 0, 0); +} + +static constexpr uint8_t node_childindex(NodeID node) { + uint64_t masked = node & node_lxyz(0, 0x0003, 0x0003, 0x0003); + return (masked >> 0) | (masked >> 14) | (masked >> 28); +} + +static constexpr ChildBits node_childbit(NodeID node) { + return child_bit(node_childindex(node)); +} + +static constexpr NodeID node_child(NodeID node, uint16_t x, uint16_t y, uint16_t z) { + int level = node_get_level(node); + return ((node & node_lxyz(0, 0xFFFF, 0xFFFF, 0xFFFF)) << 2) | node_lxyz(level - 1, x, y, z); +} + +static constexpr NodeID node_nthchild(NodeID node, int i) { + return node_child(node, (i >> 4) & 3, (i >> 2) & 3, (i >> 0) & 3); +} + +static void print_node_id(NodeID node, std::ostream *os) { + int level = node_get_level(node); + + if (level >= 8) { + (*os) << "L" << level << ":root"; + return; + } + + uint16_t x = node_get_x(node); + uint16_t y = node_get_y(node); + uint16_t z = node_get_z(node); + auto fmt = util::hex.width(4 - (level >> 1)).fill('0'); + (*os) << "L" << level << ":" << fmt.val(x) << "," << fmt.val(y) << "," << fmt.val(z); +} + +enum BBoxCheck { INSIDE_BBOX, JUST_OUTSIDE_BBOX, WAY_OUTSIDE_BBOX }; + +struct NodeInfo { + NodeID node; + BBoxCheck bbcheck; + NodeInfo(float scale, float x, float y, float z) { + float sx = (x * scale); + float sy = (y * scale); + float sz = (z * scale); + float dist = std::max({std::abs(sx), std::abs(sy), std::abs(sz)}); + if (dist >= 32768.0f) { + bbcheck = (dist > (32768.0f + 8192.0f)) ? WAY_OUTSIDE_BBOX : JUST_OUTSIDE_BBOX; + float clampx = std::min(k_hi, std::max(k_lo, sx + 32768.0f)); + float clampy = std::min(k_hi, std::max(k_lo, sy + 32768.0f)); + float clampz = std::min(k_hi, std::max(k_lo, sz + 32768.0f)); + node = node_lxyz(0, clampx, clampy, clampz); + } else { + node = node_lxyz(0, sx + 32768.0f, sy + 32768.0f, sz + 32768.0f); + bbcheck = INSIDE_BBOX; + } } }; -// Get the range of cells that includes everything in the rectangle. -// -// Gracefully handles the case that some or all of the rectangle is -// beyond the maximum cell range. In that case, it clamps to the edge -// of the cell range. -// -static CellRange rect_cell_range(float x1, float y1, float x2, float y2) { - CellRange result; - result.xlo = round_and_clamp(x1 / CELL_SCALE, -CELL_LIMIT, CELL_LIMIT); - result.ylo = round_and_clamp(y1 / CELL_SCALE, -CELL_LIMIT, CELL_LIMIT); - result.xhi = round_and_clamp(x2 / CELL_SCALE, -CELL_LIMIT, CELL_LIMIT); - result.yhi = round_and_clamp(y2 / CELL_SCALE, -CELL_LIMIT, CELL_LIMIT); - return result; -} +template +struct TreeLevel { + constexpr static int child() { return LEVEL - 1; } +}; -static int64_t cell_id(int64_t cellx, int64_t celly) { - int64_t icellx = cellx & 0xFFFFFF; - int64_t icelly = celly & 0xFFFFFF; - return 0x0001000000000000 | (icellx << 24) | (icelly << 0); -} +template <> +struct TreeLevel<0> { + constexpr static int child() { return 0; } +}; -// Get the cell ID of the specified point -static int64_t point_cell_id(float x, float y) { - float cellx = round_and_clamp(x / CELL_SCALE, -CELL_LIMIT, CELL_LIMIT); - float celly = round_and_clamp(y / CELL_SCALE, -CELL_LIMIT, CELL_LIMIT); - return cell_id(int64_t(cellx), int64_t(celly)); -} +// Class PlaneTree. Everything here is 'public', but this class +// is only visible inside this one C++ file. +class PlaneTree : public eng::opnew { +public: + void set_radius(float r); + + using NodeID = uint64_t; + using ChildBits = uint64_t; + using IdVector = util::IdVector; -void PlaneMap::remove(const eng::string &plane, int64_t cellid, PlaneItem *client) { - auto piter = planes_.find(plane); - if (piter == planes_.end()) { - return; + // The PlaneMap that this tree is a part of. + PlaneMap *planemap_; + + // The name of this plane. + eng::string plane_; + + // Internal nodes in the tree just have bits indicating + // which children exist. + eng::bytell_hash_map internal_nodes_; + + // Leaf nodes in the tree contain a doubly-linked + // intrusive ring. + eng::bytell_hash_map leaf_nodes_; + + // The radius of the bounding box. + double radius_; + + // A conversion factor to convert float coordinates to + // integral coordinates. Equal to 32k / radius. + float scale_; + + // total number of items in the planetree. + int total_count_; + + // total number of outliers in the planetree. Outliers are + // classified as just outside or way outside the bbox. + int just_outside_bbox_; + int way_outside_bbox_; + +public: + // The following state is initialized whenever we do a scan. + // It is only relevant during the scan. + const PlaneScan *scan_config_; + IdVector *scan_result_; + util::XYZ scan_lo_; + util::XYZ scan_hi_; + util::XYZ scan_invradius_; + int scan_bbxlo_[9], scan_bbxhi_[9]; + int scan_bbylo_[9], scan_bbyhi_[9]; + int scan_bbzlo_[9], scan_bbzhi_[9]; + +public: + ChildBits get_internal_node(NodeID id) const { + auto iter = internal_nodes_.find(id); + if (iter == internal_nodes_.end()) return 0; + return iter->second; } - Plane &p = piter->second; - auto liter = p.find(cellid); - if (liter == p.end()) { - return; - } - EltVec &l = liter->second; - l.erase(std::remove(l.begin(), l.end(), client), l.end()); - if (l.empty()) { - p.erase(liter); - } - if (p.empty()) { - planes_.erase(piter); - } -} -void PlaneMap::insert(const eng::string &plane, int64_t cellid, PlaneItem *client) { - Plane &p = planes_[plane]; - EltVec &l = p[cellid]; - l.push_back(client); -} + PlaneItem *get_leaf_node(NodeID id) const { + auto iter = leaf_nodes_.find(id); + if (iter == leaf_nodes_.end()) return 0; + return iter->second; + } + + // Untrack all planeitems. This is for unit testing and for destructors. We + // don't use PlaneItem::untrack, because that would create problems with + // removing items from a list while iterating over that list. + void untrack_all() { + for (auto &l : leaf_nodes_) { + PlaneItem *first = l.second; + PlaneItem *pi = first; + assert(pi != nullptr); + while (true) { + PlaneItem *next = pi->next_; + pi->tree_= nullptr; + pi = next; + if (pi == first) break; + } + } + leaf_nodes_.clear(); + internal_nodes_.clear(); + } -void PlaneItem::set_pos(const eng::string &plane, float x, float y, float z) { - int64_t old_cell = point_cell_id(x_, y_); - int64_t new_cell = point_cell_id(x, y); + // This just sets the radius. + // It verifies that the tree is empty first. + void set_radius_of_empty_tree(float r) { + assert(total_count_ == 0); + radius_ = r; + scale_ = 32768.0 / r; + } - // Update the grid. - if (pmap_ != 0) { - if ((plane_ != plane) || (old_cell != new_cell)) { - pmap_->remove(plane_, old_cell, this); - pmap_->insert(plane, new_cell, this); + // Get a PlaneTree by plane name. + static PlaneTree *get(PlaneMap *pmap, const eng::string &plane) { + std::unique_ptr &result = pmap->planes_[plane]; + if (result == nullptr) { + result.reset(new PlaneTree(pmap, plane)); + } + return result.get(); + } + + // Remove an item from the specified leaf node. + // Returns true if we removed the last item from the leaf. + bool remove_planeitem_from_leaf(NodeID node, PlaneItem *item) { + if (item->next_ == item) { + leaf_nodes_.erase(node); + // Note: these next two assignments are only needed for sanity checking. + item->next_ = nullptr; + item->prev_ = nullptr; + return true; + } else { + item->prev_->next_ = item->next_; + item->next_->prev_ = item->prev_; + // Note: these next two assignments are only needed for sanity checking. + item->next_ = nullptr; + item->prev_ = nullptr; + return false; } } - // Update the client position. - plane_ = plane; - x_ = x; - y_ = y; - z_ = z; -} - -void PlaneItem::untrack() { - if (pmap_ != 0) { - pmap_->remove(plane_, point_cell_id(x_, y_), this); - pmap_ = 0; + // Insert an item into the specified leaf node. + // Returns true if we inserted the first item into the leaf. + bool insert_planeitem_into_leaf(NodeID node, PlaneItem *item) { + PlaneItem *&newcell = leaf_nodes_[node]; + if (newcell == nullptr) { + newcell = item; + item->next_ = item; + item->prev_ = item; + return true; + } else { + PlaneItem *next = newcell; + PlaneItem *prev = newcell->prev_; + item->next_ = next; + item->prev_ = prev; + prev->next_ = item; + next->prev_ = item; + return false; + } } -} -void PlaneItem::track(PlaneMap *pmap) { - if (pmap_ != pmap) { - untrack(); - pmap->insert(plane_, point_cell_id(x_, y_), this); - pmap_ = pmap; + // Update the parent to reflect the fact that the child was added. + // This will propagage all the way up the tree. + void insert_child_into_childbits(NodeID child) { + uint8_t childlevel = node_get_level(child); + uint8_t parentlevel = childlevel + 1; + NodeID parent = node_parent(child); + ChildBits &inode = internal_nodes_[parent]; + bool waszero = (inode == 0); + inode |= node_childbit(child); + if (waszero) { + if (parentlevel < 8) insert_child_into_childbits(parent); + } } -} -PlaneItem::PlaneItem() : pmap_(NULL), x_(0.0), y_(0.0), z_(0.0) { -} + // Update the parent to reflect the fact that the child was removed. + // This will propagage all the way up the tree. + void remove_child_from_childbits(NodeID child) { + uint8_t childlevel = node_get_level(child); + uint8_t parentlevel = childlevel + 1; + NodeID parent = node_parent(child); + auto iter = internal_nodes_.find(parent); + assert(iter != internal_nodes_.end()); + iter->second &= (~node_childbit(child)); + if (iter->second == 0) { + internal_nodes_.erase(iter); + if (parentlevel < 8) remove_child_from_childbits(parent); + } + } -PlaneMap::PlaneMap() { -} + // Update the counters to reflect the removal of one item from the tree. + void decrement_planeitem_counters(BBoxCheck bbcheck) { + total_count_ -= 1; + if (bbcheck == JUST_OUTSIDE_BBOX) just_outside_bbox_ -= 1; + if (bbcheck == WAY_OUTSIDE_BBOX) way_outside_bbox_ -= 1; + } -PlaneItem::~PlaneItem() { - untrack(); -} + // Update the counters to reflect the insertion of one item into the tree. + void increment_planeitem_counters(BBoxCheck bbcheck) { + total_count_ += 1; + if (bbcheck == JUST_OUTSIDE_BBOX) just_outside_bbox_ += 1; + if (bbcheck == WAY_OUTSIDE_BBOX) way_outside_bbox_ += 1; + } -PlaneMap::~PlaneMap() { - for (const auto &p : planes_) { - for (const auto &l : p.second) { - for (PlaneItem *i : l.second) { - i->pmap_ = NULL; + // Remove a planeitem from whatever tree it is in, preserving + // all invariants. The planeitem ends up being an untracked PlaneItem. + static void remove_planeitem(PlaneItem *item) { + PlaneTree *tree = item->tree_; + assert(tree != nullptr); + NodeInfo info(tree->scale_, item->x_, item->y_, item->z_); + tree->decrement_planeitem_counters(info.bbcheck); + if (tree->remove_planeitem_from_leaf(info.node, item)) { + tree->remove_child_from_childbits(info.node); + } + item->tree_ = nullptr; + } + + // Insert a planeitem into whatever tree is specified, preserving + // all invariants. The planeitem must be an untracked PlaneItem. + void insert_planeitem(PlaneItem *item) { + PlaneTree *tree = this; + assert(item->tree_ == nullptr); + NodeInfo info(tree->scale_, item->x_, item->y_, item->z_); + tree->increment_planeitem_counters(info.bbcheck); + if (tree->insert_planeitem_into_leaf(info.node, item)) { + tree->insert_child_into_childbits(info.node); + } + item->tree_ = tree; + item->plane_ = tree->plane_; + } + + void print_indented_internal_node(NodeID node, ChildBits cb, std::ostream *os) { + int level = node_get_level(node); + int indent = 8 - level; + (*os) << "|"; + for (int i = 0; i < indent; i++) (*os) << " "; + print_node_id(node, os); + if ((cb == 0) && (level != 8)) { + (*os) << " (invalid empty node)"; + } + } + + void print_indented_leaf_node(NodeID node, PlaneItem *first, std::ostream *os) { + (*os) << "| "; + print_node_id(node, os); + (*os) << " "; + util::IdVector ids; + collect_planeitem_ids(first, &ids); + std::sort(ids.begin(), ids.end()); + util::print_id_vector(ids, os); + } + + void print_tree_r(NodeID node, std::ostream *os) { + int level = node_get_level(node); + if (level == 0) { + print_indented_leaf_node(node, get_leaf_node(node), os); + } else { + ChildBits cb = get_internal_node(node); + if ((level & 1) == 0) { + print_indented_internal_node(node, cb, os); + } + for (int i = 0; i < 64; i++) { + if (cb & child_bit(i)) { + NodeID child = node_nthchild(node, i); + assert(node_childindex(child) == i); + print_tree_r(child, os); + } } } } -} -PlaneMap::EltVec PlaneMap::get_cell(const eng::string &plane, int64_t cellid) const { - PlaneMap::EltVec result; - auto piter = planes_.find(plane); - if (piter != planes_.end()) { - const Plane &p = piter->second; - auto liter = p.find(cellid); - if (liter != p.end()) { - result = liter->second; + // The final filtering step sometimes uses the inverse of the + // radius. In the case that the radius is 0, we want to use a huge + // number for the inverse radius, but not infinity, because using infinity + // would result in the final filtering step calculating (inf*0). + // In the case that the radius is infinite, we want to use zero for the + // inverse radius. + static float inverse_radius(float f) { + if (f == 0) return std::numeric_limits::max(); + if (std::isinf(f)) return 0; + return 1.0f / f; + } + + // Given a PlaneScan, calculate the search bboxes, + // and all the other related configuration data. + void calculate_search_bboxes(const PlaneScan &sc) { + scan_config_ = ≻ + scan_lo_ = sc.center_ - sc.radius_; + scan_hi_ = sc.center_ + sc.radius_; + + scan_invradius_.x = inverse_radius(sc.radius_.x); + scan_invradius_.y = inverse_radius(sc.radius_.y); + scan_invradius_.z = inverse_radius(sc.radius_.z); + + // Convert the scan's bounding box to integral coordinates. + NodeInfo bblo(scale_, scan_lo_.x, scan_lo_.y, scan_lo_.z); + NodeInfo bbhi(scale_, scan_hi_.x, scan_hi_.y, scan_hi_.z); + + // Calculate the bounding box at each level of the tree. + NodeID ibblo = bblo.node; + NodeID ibbhi = bbhi.node; + for (int i = 0; i <= 8; i++) { + scan_bbxlo_[i] = node_get_x(ibblo); + scan_bbylo_[i] = node_get_y(ibblo); + scan_bbzlo_[i] = node_get_z(ibblo); + scan_bbxhi_[i] = node_get_x(ibbhi); + scan_bbyhi_[i] = node_get_y(ibbhi); + scan_bbzhi_[i] = node_get_z(ibbhi); + ibblo = node_parent(ibblo); + ibbhi = node_parent(ibbhi); } } - return result; -} -int PlaneMap::total_cells() const { - int total = 0; - for (const auto &p : planes_) { - total += p.second.size(); - } - return total; -} + // Calculate the size of the search bboxes. + int64_t scan_xsize(int i) const { return 1 + scan_bbxhi_[i] - scan_bbxlo_[i]; } + int64_t scan_ysize(int i) const { return 1 + scan_bbyhi_[i] - scan_bbylo_[i]; } + int64_t scan_zsize(int i) const { return 1 + scan_bbzhi_[i] - scan_bbzlo_[i]; } -PlaneMap::IdVector PlaneMap::scan_radius_unsorted(const eng::string &plane, float x, float y, float radius, bool exclude_nowhere, int64_t special, bool omit) const { - PlaneMap::IdVector result; - if ((special != 0)&&(!omit)) { - result.push_back(special); + eng::string search_bboxes_debug_string(const PlaneScan &scan) { + calculate_search_bboxes(scan); + eng::ostringstream oss; + for (int i = 8; i >= 0; i -= 2) { + auto fmt = util::hex.width((8 - i) / 2); + oss << "|Level " << i << " "; + oss << fmt.val(scan_bbxlo_[i]) << "," + << fmt.val(scan_bbylo_[i]) << "," + << fmt.val(scan_bbzlo_[i]) << " - " + << fmt.val(scan_bbxhi_[i]) << "," + << fmt.val(scan_bbyhi_[i]) << "," + << fmt.val(scan_bbzhi_[i]); + } + return oss.str(); } - if (exclude_nowhere && (plane == "nowhere")) { - return result; + + static inline void scan_push_id(int64_t id, int64_t near, IdVector *result) { + if (id != near) { + result->push_back(id); + } } - auto piter = planes_.find(plane); - if (piter != planes_.end()) { - const Plane &p = piter->second; - CellRange range = rect_cell_range(x - radius, y - radius, x + radius, y + radius); - float radsq = radius*radius; - for (int cy = range.ylo; cy <= range.yhi; cy++) { - for (int cx = range.xlo; cx <= range.xhi; cx++) { - auto liter = p.find(cell_id(cx, cy)); - if (liter != p.end()) { - for (PlaneItem *client : liter->second) { - if (util::distance_squared(client->x(), client->y(), x, y) <= radsq) { - if (client->id() != special) { - result.push_back(client->id()); - } + + void scan_planeitem(PlaneItem *pi) { + switch (scan_config_->shape_) { + case PlaneScan::BOX: { + if ((pi->x() >= scan_lo_.x) && (pi->x() <= scan_hi_.x) && + (pi->y() >= scan_lo_.y) && (pi->y() <= scan_hi_.y) && + (pi->z() >= scan_lo_.z) && (pi->z() <= scan_hi_.z)) { + scan_push_id(pi->id(), scan_config_->near_, scan_result_); + } + break; + } + case PlaneScan::SPHERE: { + float dx = (pi->x() - scan_config_->center_.x) * scan_invradius_.x; + float dy = (pi->y() - scan_config_->center_.y) * scan_invradius_.y; + float dz = (pi->z() - scan_config_->center_.z) * scan_invradius_.z; + if (dx*dx + dy*dy + dz*dz <= 1.0) { + scan_push_id(pi->id(), scan_config_->near_, scan_result_); + } + break; + } + case PlaneScan::CYLINDER: { + if ((pi->z() >= scan_lo_.z) && (pi->z() <= scan_hi_.z)) { + float dx = (pi->x() - scan_config_->center_.x) * scan_invradius_.x; + float dy = (pi->y() - scan_config_->center_.y) * scan_invradius_.y; + if (dx*dx + dy*dy <= 1.0) { + scan_push_id(pi->id(), scan_config_->near_, scan_result_); + } + } + break; + } + } + } + + // Recursive part of the planetree scan. + // Note: template expansion terminates because + // TreeLevel<0>::child returns zero again. + template + void scan_node(NodeID node, std::ostream *debug) { + if (LEVEL == 0) { + auto iter = leaf_nodes_.find(node); + assert (iter != leaf_nodes_.end()); + PlaneItem *first = iter->second; + assert(first != nullptr); + if (debug != nullptr) { + print_indented_leaf_node(node, first, debug); + } + PlaneItem *pi = first; + while (true) { + PlaneItem *next = pi->next_; + scan_planeitem(pi); + if (next == first) break; + pi = next; + } + } else { + constexpr int CHILDLEVEL = TreeLevel::child(); + NodeID firstchild = node_nthchild(node, 0); + + int xlo = std::max(0, scan_bbxlo_[CHILDLEVEL] - int(node_get_x(firstchild))); + int ylo = std::max(0, scan_bbylo_[CHILDLEVEL] - int(node_get_y(firstchild))); + int zlo = std::max(0, scan_bbzlo_[CHILDLEVEL] - int(node_get_z(firstchild))); + int xhi = std::min(3, scan_bbxhi_[CHILDLEVEL] - int(node_get_x(firstchild))); + int yhi = std::min(3, scan_bbyhi_[CHILDLEVEL] - int(node_get_y(firstchild))); + int zhi = std::min(3, scan_bbzhi_[CHILDLEVEL] - int(node_get_z(firstchild))); + + auto iter = internal_nodes_.find(node); + assert (iter != internal_nodes_.end()); + ChildBits cb = iter->second; + assert (cb != 0); + if (debug != nullptr) { + print_indented_internal_node(node, cb, debug); + } + + for (int x = xlo; x <= xhi; x++) { + for (int y = ylo; y <= yhi; y++) { + for (int z = zlo; z <= zhi; z++) { + if (cb & child_bit(x, y, z)) { + NodeID child = node_child(node, x, y, z); + scan_node(child, debug); } } } } } } - return result; + + // Scan a planetree. + void scan(const PlaneScan &sc, IdVector *result, std::ostream *debug) { + calculate_search_bboxes(sc); + scan_result_ = result; + + // We must only call 'scan_node' on nodes that actually exist. + // So we check if the tree is empty, and if so, we don't scan the + // root node. + if (!internal_nodes_.empty()) { + NodeID root = node_lxyz(8, 0, 0, 0); + scan_node<8>(root, debug); + } + } + + eng::string scan_steps_debug_string(const PlaneScan &sc) { + eng::ostringstream oss; + IdVector result; + scan(sc, &result, &oss); + oss << "|Result: "; + std::sort(result.begin(), result.end()); + util::print_id_vector(result, &oss); + return oss.str(); + } + + void collect_planeitem_ids(PlaneItem *first, IdVector *ids) { + if (first != nullptr) { + PlaneItem *pi = first; + while (true) { + PlaneItem *next = pi->next_; + ids->push_back(pi->id()); + if (next == first) break; + pi = next; + } + } + } + + eng::string tree_debug_string() { + eng::ostringstream oss; + print_tree_r(node_lxyz(8,0,0,0), &oss); + return oss.str(); + } + + eng::string outliers_debug_string() { + eng::ostringstream oss; + oss << "total:" << total_count_ << " justout:" << just_outside_bbox_ << " wayout:" << way_outside_bbox_; + return oss.str(); + } + + // Construct a PlaneTree. + PlaneTree(PlaneMap *pmap, const eng::string &plane) { + planemap_ = pmap; + plane_ = plane; + total_count_ = 0; + just_outside_bbox_ = 0; + way_outside_bbox_ = 0; + set_radius_of_empty_tree(pmap->default_radius_); + } + + // Destructor: the PlaneTree doesn't own the PlaneItems, so it doesn't + // delete them, but it needs to untrack all the PlaneItems. + ~PlaneTree() { + untrack_all(); + } +}; + +void PlaneItem::track(PlaneMap *pmap) { + // If we're already in a PlaneMap, and it's not the + // PlaneMap we want to be in, remove from the old PlaneMap. + if ((tree_ != nullptr) && (tree_->planemap_ != pmap)) { + PlaneTree::remove_planeitem(this); + } + + // If we're supposed to be in a PlaneMap, and we're not + // already in the PlaneMap, insert it. + if ((tree_ == nullptr) && (pmap != nullptr)) { + PlaneTree::get(pmap, plane_)->insert_planeitem(this); + } } -PlaneMap::IdVector PlaneMap::scan_radius(const eng::string &plane, float x, float y, float radius, bool exclude_nowhere, int64_t special, bool omit) const { - PlaneMap::IdVector result = scan_radius_unsorted(plane, x, y, radius, exclude_nowhere, special, omit); - if ((special != 0)&&(!omit)) { - std::sort(result.begin() + 1, result.end()); - } else { - std::sort(result.begin(), result.end()); +void PlaneItem::set_pos(const eng::string &plane, float x, float y, float z) { + // If we're not in a PlaneMap, nothing to do but set the variables. + if (tree_ == nullptr) { + plane_ = plane; + x_ = x; + y_ = y; + z_ = z; + return; + } + + // When moving within a plane (not warping), use set_xyz, which is faster. + if (plane_ == plane) { + set_xyz(x, y, z); + return; + } + + // We're warping from one plane to another. That means we're removing + // ourself from one planetree and inserting ourself into a different one. + PlaneTree *newtree = PlaneTree::get(tree_->planemap_, plane); + PlaneTree::remove_planeitem(this); + x_ = x; + y_ = y; + z_ = z; + newtree->insert_planeitem(this); +} + +void PlaneItem::set_xyz(float x, float y, float z) { + // If we're not in a PlaneMap, nothing to do but set the variables. + if (tree_ == nullptr) { + x_ = x; + y_ = y; + z_ = z; + return; + } + + // We could implement this function using 'PlaneTree::remove_planeitem' + // and 'PlaneTree::insert_planeitem', which would be less code, but it + // would also be slower. + NodeInfo old_cell(tree_->scale_, x_, y_, z_); + NodeInfo new_cell(tree_->scale_, x, y, z); + + // Update the variables. + x_ = x; + y_ = y; + z_ = z; + + // Update the outliers counters (unlikely). + if (old_cell.bbcheck != new_cell.bbcheck) { + tree_->decrement_planeitem_counters(old_cell.bbcheck); + tree_->increment_planeitem_counters(new_cell.bbcheck); + } + + // If we have changed cells, update the tree. + // We have to remove the child from the old leaf before inserting + // it into the new leaf, because the 'next' and 'prev' pointers are + // intrusive and we need them to be unused to do the insert. + // However, inserting the child into the childbits first is faster + // and poses no problems. + if (new_cell.node != old_cell.node) { + bool leaf_removed = tree_->remove_planeitem_from_leaf(old_cell.node, this); + bool leaf_created = tree_->insert_planeitem_into_leaf(new_cell.node, this); + if (leaf_created) tree_->insert_child_into_childbits(new_cell.node); + if (leaf_removed) tree_->remove_child_from_childbits(old_cell.node); + } +} + +PlaneItem::PlaneItem() { + id_ = 0; + tree_ = nullptr; + next_ = nullptr; + prev_ = nullptr; + x_ = y_ = z_ = 0.0; +} + +PlaneItem::~PlaneItem() { + untrack(); +} + +PlaneMap::PlaneMap() : default_radius_(32768.0) {} +PlaneMap::~PlaneMap() {} + + +IdVector PlaneMap::scan(const PlaneScan &sc) const { + IdVector result; + int startpos = 0; + if (sc.near_ != 0) { + if (sc.include_near_) { + result.push_back(sc.near_); + startpos = 1; + } + } + + if (sc.omit_nowhere_ && (sc.plane_ == "nowhere")) { + return result; + } + + auto piter = planes_.find(sc.plane_); + if (piter != planes_.end()) { + const std::unique_ptr &tree = piter->second; + tree->scan(sc, &result, nullptr); + } + + if (sc.sorted_) { + std::sort(result.begin() + startpos, result.end()); } return result; } +eng::string PlaneMap::tree_debug_string(const eng::string &plane) { + return PlaneTree::get(this, plane)->tree_debug_string(); +} + +eng::string PlaneMap::outliers_debug_string(const eng::string &plane) { + return PlaneTree::get(this, plane)->outliers_debug_string(); +} + +eng::string PlaneMap::search_bboxes_debug_string(const PlaneScan &scan) { + return PlaneTree::get(this, scan.plane_)->search_bboxes_debug_string(scan); +} + +eng::string PlaneMap::scan_steps_debug_string(const PlaneScan &scan) { + return PlaneTree::get(this, scan.plane_)->scan_steps_debug_string(scan); +} + +void PlaneMap::untrack_all() { + for (const auto &pair : planes_) { + pair.second->untrack_all(); + } +} + +eng::string PlaneScan::configure(const LuaStack &LS0, LuaSlot config) { + LuaVar val, vx, vy, vz; + LuaStack LS(LS0.state(), val, vx, vy, vz); + + if (!LS.istable(config)) { + return "scan configuration is not a table"; + } + + bool have_plane = false; + bool have_center = false; + bool have_radius = false; + bool have_shape = false; + bool have_near = false; + int parameters = 0; + + LS.rawget(val, config, "plane"); + if (!LS.isnil(val)) { + if (!LS.isstring(val)) { + return "scan configuration: 'plane' must be a string"; + } + plane_ = LS.ckstring(val); + have_plane = true; + parameters += 1; + } + + LS.rawget(vx, config, "centerx"); + LS.rawget(vy, config, "centery"); + LS.rawget(vz, config, "centerz"); + if ((!LS.isnil(vx)) || (!LS.isnil(vy)) || (!LS.isnil(vz))) { + if (!LS.isnumber(vx)) { + return "scan configuration: 'centerx' must be a number"; + } + if (!LS.isnumber(vy)) { + return "scan configuration: 'centery' must be a number"; + } + if (!LS.isnumber(vz)) { + return "scan configuration: 'centerz' must be a number"; + } + center_.x = LS.cknumber(vx); + center_.y = LS.cknumber(vy); + center_.z = LS.cknumber(vz); + have_center = true; + parameters += 3; + } + + LS.rawget(val, config, "radius"); + if (!LS.isnil(val)) { + if (!LS.isnumber(val)) { + return "scan configuration: 'radius' must be a number"; + } + radius_.x = LS.cknumber(val); + radius_.y = radius_.z = radius_.x; + have_radius = true; + parameters += 1; + } + + LS.rawget(vx, config, "radiusx"); + LS.rawget(vy, config, "radiusy"); + LS.rawget(vz, config, "radiusz"); + if ((!LS.isnil(vx)) || (!LS.isnil(vy)) || (!LS.isnil(vz))) { + if (!LS.isnumber(vx)) { + return "scan configuration: 'radiusx' must be a number"; + } + if (!LS.isnumber(vy)) { + return "scan configuration: 'radiusy' must be a number"; + } + if (!LS.isnumber(vz)) { + return "scan configuration: 'radiusz' must be a number"; + } + if (have_radius) { + return "scan configuration: specified both 'radius' and 'radiusx'"; + } + radius_.x = LS.cknumber(vx); + radius_.y = LS.cknumber(vy); + radius_.z = LS.cknumber(vz); + have_radius = true; + parameters += 3; + } + + LS.rawget(val, config, "shape"); + if (!LS.isnil(val)) { + if (!LS.isstring(val)) { + return "scan configuration: 'shape' must be a string"; + } + eng::string shape = LS.ckstring(val); + if (shape == "box") { + shape_ = BOX; + } else if (shape == "sphere") { + shape_ = SPHERE; + } else if (shape == "cylinder") { + shape_ = CYLINDER; + } else { + return util::ss("scan configuration: unknown shape ", shape); + } + have_shape = true; + parameters += 1; + } + + LS.rawget(val, config, "near"); + if (!LS.isnil(val)) { + int64_t id = LS.tanid(val); + if (id == 0) { + return "scan configuration: 'near' must be a tangible"; + } + if (have_center) { + return "scan configuration: specified both 'center' and 'near'"; + } + if (have_plane) { + return "scan configuration: specified both 'plane' and 'near'"; + } + near_ = id; + have_center = true; + have_plane = true; + have_near = true; + parameters += 1; + } + + LS.rawget(val, config, "include"); + if (!LS.isnil(val)) { + if (!LS.isboolean(val)) { + return "scan configuration: 'include' must be a boolean"; + } + if (!have_near) { + return "scan configuration: 'include' specified without 'near'"; + } + include_near_ = LS.ckboolean(val); + parameters += 1; + } + + LS.rawget(val, config, "wholeplane"); + if (!LS.isnil(val)) { + if (!LS.isstring(val)) { + return "scan configuration: 'wholeplane' must be a string"; + } + if (have_plane || have_center || have_radius || have_shape) { + return "scan configuration: do not specify plane, center, shape, or radius with 'wholeplane'"; + } + plane_ = LS.ckstring(val); + set_whole_plane(); + have_plane = true; + have_center = true; + have_radius = true; + have_shape = true; + parameters += 1; + } + + if (lua_nkeys(LS.state(), config.index()) != parameters) { + return "scan configuration: unrecognized parameters in table"; + } + + if (!have_plane) { + return "scan configuration: did not specify plane"; + } + if (!have_radius) { + return "scan configuration: did not specify radius"; + } + if (!have_center) { + return "scan configuration: did not specify center"; + } + if (!have_shape) { + shape_ = SPHERE; // default value. + } + return ""; +} + +eng::string PlaneScan::debug_string() const { + eng::ostringstream oss; + oss << "plane:" << plane_ << " center:" << center_ << " radius:" << radius_; + if (shape_ == BOX) oss << " shape:box"; + else if (shape_ == SPHERE) oss << " shape:sphere"; + else if (shape_ == CYLINDER) oss << " shape:cylinder"; + else oss << " shape:unknown"; + if (near_ != 0) { + oss << " near:" << near_ << " include:" << include_near_; + } + if (omit_nowhere_) { + oss << " omit_nowhere:true"; + } + if (!sorted_) { + oss << " sorted:false"; + } + return oss.str(); +} + +// The default radius is set such that float coordinates map directly to +// integer coordinates, with an offset of 0x8000. This makes unit testing +// a lot easier. +// LuaDefine(unittests_planemap, "", "some unit tests") { - float SC = CELL_SCALE; - float E = CELL_SCALE * 0.4; - int LO = -CELL_LIMIT; - int HI = CELL_LIMIT; PlaneMap pm; - PlaneItem pia, pib; - PlaneMap::EltVec elts; - PlaneMap::IdVector ids; + PlaneItem pi123, pi456; + PlaneScan scan; + pi123.set_id(123); + pi456.set_id(456); - // Simple test. - LuaAssert(L, rect_cell_range(-7*SC, -15*SC, 87*SC, 21*SC).equal(-7, -15, 87, 21)); - - // Adding an epsilon doesn't change result, if epsilon is less than half of cell scale. - LuaAssert(L, rect_cell_range(-7*SC+E, -15*SC+E, 87*SC-E, 21*SC-E).equal(-7, -15, 87, 21)); + // Test that PlaneItems can be manipulated when they're not + // yet tracking a PlaneMap. + pi123.set_pos("p", 0x38, 0x16, 0x87); + LuaAssert(L, pi123.plane() == "p"); + LuaAssert(L, pi123.x() == 0x38); + LuaAssert(L, pi123.y() == 0x16); + LuaAssert(L, pi123.z() == 0x87); - // Rectangle that crosses the high end of the range. - LuaAssert(L, rect_cell_range((HI-7)*SC, (HI-5)*SC, (HI+3)*SC, (HI+6)*SC).equal(HI-7, HI-5, HI, HI)); - - // Rectangle that exceeds the high end of the range. - LuaAssert(L, rect_cell_range((HI+7)*SC, (HI+5)*SC, (HI+15)*SC, (HI+12)*SC).equal(HI, HI, HI, HI)); + // TESTS OF TREE MANIPULATION FOLLOW. - // Rectangle that crosses the low end of the range. - LuaAssert(L, rect_cell_range((LO-7)*SC, (LO-5)*SC, (LO+3)*SC, (LO+4)*SC).equal(LO, LO, LO+3, LO+4)); + // Test track. + pi123.set_pos("p", 0x38, 0x16, 0x87); + pi123.track(&pm); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,8,8" + "| L4:80,80,80" + "| L2:803,801,808" + "| L0:8038,8016,8087 123"); + + // Test untrack. + pi123.untrack(); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root"); - // Rectangle that exceeds the low end of the range. - LuaAssert(L, rect_cell_range((LO-15)*SC, (LO-17)*SC, (LO-7)*SC, (LO-5)*SC).equal(LO, LO, LO, LO)); - - // Simple test. - LuaAssert(L, point_cell_id(-7*SC, 15*SC) == cell_id(-7, 15)); - - // Adding epsilon doesn't change the result if less than half cell scale. - LuaAssert(L, point_cell_id(-7*SC+E, 15*SC+E) == cell_id(-7, 15)); - - // Right at the top edge of the range. - LuaAssert(L, point_cell_id(HI*SC, HI*SC) == cell_id(HI, HI)); - - // Right at the bottom edge of the range. - LuaAssert(L, point_cell_id(LO*SC, LO*SC) == cell_id(LO, LO)); - - // Beyond various edges. - LuaAssert(L, point_cell_id((LO-1)*SC, 0) == point_cell_id(LO*SC, 0)); - LuaAssert(L, point_cell_id((HI+1)*SC, 0) == point_cell_id(HI*SC, 0)); - LuaAssert(L, point_cell_id(0, (LO-1)*SC) == point_cell_id(0, LO*SC)); - LuaAssert(L, point_cell_id(0, (HI+1)*SC) == point_cell_id(0, HI*SC)); - - // Test using the insert function. - pm.clear(); - LuaAssert(L, pm.total_cells() == 0); - pm.insert("foo", 12345, &pia); - LuaAssert(L, pm.total_cells() == 1); - pm.insert("foo", 12345, &pib); - LuaAssert(L, pm.total_cells() == 1); - elts = pm.get_cell("foo", 12345); - LuaAssert(L, elts.size() == 2); - LuaAssert(L, elts[0] == &pia); - LuaAssert(L, elts[1] == &pib); - - // Test the remove function. - pm.remove("foo", 12345, &pia); - LuaAssert(L, pm.total_cells() == 1); - elts = pm.get_cell("foo", 12345); - LuaAssert(L, elts.size() == 1); - LuaAssert(L, elts[0] == &pib); - pm.remove("foo", 12345, &pib); - LuaAssert(L, pm.total_cells() == 0); + // Track two items at a time, not in the same cell. + pi456.set_pos("p", 0x12, 0x17, 0xAC); + pi123.track(&pm); + pi456.track(&pm); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,8,8" + "| L4:80,80,80" + "| L2:801,801,80a" + "| L0:8012,8017,80ac 456" + "| L2:803,801,808" + "| L0:8038,8016,8087 123"); - // Try moving a plane item around without it being tracked to a grid. - pia.set_pos("foo", 3, 4, 5); - LuaAssert(L, pia.plane() == "foo"); - LuaAssert(L, pia.x() == 3.0); - LuaAssert(L, pia.y() == 4.0); - LuaAssert(L, pia.z() == 5.0); + // Move one of the items into the same cell as the other. + pi456.set_xyz(0x38, 0x16, 0x87); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,8,8" + "| L4:80,80,80" + "| L2:803,801,808" + "| L0:8038,8016,8087 123,456"); - // Attach pia to the grid. This should record it. - pm.clear(); - pia.track(&pm); - elts = pm.get_cell("foo", point_cell_id(3.0, 4.0)); - LuaAssert(L, elts.size() == 1); - LuaAssert(L, elts[0] == &pia); - - // Unattach pia from the grid. This should unrecord it. - pia.untrack(); - LuaAssert(L, pm.total_cells() == 0); + // Move item 456 back out of the cell. + pi456.set_xyz(0x27, 0x11, 0x31); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,8,8" + "| L4:80,80,80" + "| L2:802,801,803" + "| L0:8027,8011,8031 456" + "| L2:803,801,808" + "| L0:8038,8016,8087 123"); - // Reattach pia to the grid, then move it. - pia.track(&pm); - LuaAssert(L, pm.total_cells() == 1); - pia.set_pos("bar", 1000.0, 1000.0, 0.0); - LuaAssert(L, pm.total_cells() == 1); - elts = pm.get_cell("bar", point_cell_id(1000.0, 1000.0)); - LuaAssert(L, elts.size() == 1); - LuaAssert(L, elts[0] == &pia); + // Move item 123 to follow 456. + pi123.set_xyz(0x27, 0x11, 0x31); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,8,8" + "| L4:80,80,80" + "| L2:802,801,803" + "| L0:8027,8011,8031 123,456"); + + // TESTS OF OUTLIER CLAMPING FOLLOW. - // Insert the four elements, then test the scan function. - pib.track(&pm); - pia.set_id(123); - pib.set_id(456); - pib.set_pos("bar", 1100.0, 1000.0, 0.0); - ids = pm.scan_radius("bar", 1000.0, 1000.0, 1.0, false, 0, false); - LuaAssert(L, ids.size() == 1); - LuaAssert(L, ids[0] == 123); - ids = pm.scan_radius("bar", 1000.0, 1000.0, 99.9, false, 0, false); - LuaAssert(L, ids.size() == 1); - LuaAssert(L, ids[0] == 123); - ids = pm.scan_radius("bar", 1000.0, 1000.0, 100.0, false, 0, false); - LuaAssert(L, ids.size() == 2); - LuaAssert(L, ids[0] == 123); - LuaAssert(L, ids[1] == 456); + // Move item 456 close to, but not quite on the positive edge. + pi123.untrack(); + pi456.set_xyz(0x23, 0x7FFE, 0x27); + LuaAssertStrEq(L, pm.outliers_debug_string("p"), "total:1 justout:0 wayout:0"); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,f,8" + "| L4:80,ff,80" + "| L2:802,fff,802" + "| L0:8023,fffe,8027 456"); + + // Move item 456 so that it's on the positive edge, but not an outlier. + pi456.set_xyz(0x23, 0x7FFF, 0x27); + LuaAssertStrEq(L, pm.outliers_debug_string("p"), "total:1 justout:0 wayout:0"); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,f,8" + "| L4:80,ff,80" + "| L2:802,fff,802" + "| L0:8023,ffff,8027 456"); + + // Move item 456 so that it's even closer to the positive edge, but not an outlier. + pi456.set_xyz(0x23, 0x7FFF + 0.99, 0x27); + LuaAssertStrEq(L, pm.outliers_debug_string("p"), "total:1 justout:0 wayout:0"); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,f,8" + "| L4:80,ff,80" + "| L2:802,fff,802" + "| L0:8023,ffff,8027 456"); + + // Move item 456 so that it's just barely a positive outlier. + pi456.set_xyz(0x23, 0x8000, 0x27); + LuaAssertStrEq(L, pm.outliers_debug_string("p"), "total:1 justout:1 wayout:0"); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,f,8" + "| L4:80,ff,80" + "| L2:802,fff,802" + "| L0:8023,ffff,8027 456"); + + // Move item 456 so that it's considerably past the positive edge. + pi456.set_xyz(0x23, 0x8048, 0x27); + LuaAssertStrEq(L, pm.outliers_debug_string("p"), "total:1 justout:1 wayout:0"); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,f,8" + "| L4:80,ff,80" + "| L2:802,fff,802" + "| L0:8023,ffff,8027 456"); + + // Move item 456 so that it's way past the positive edge. + pi456.set_xyz(0x23, 0x83748, 0x27); + LuaAssertStrEq(L, pm.outliers_debug_string("p"), "total:1 justout:0 wayout:1"); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,f,8" + "| L4:80,ff,80" + "| L2:802,fff,802" + "| L0:8023,ffff,8027 456"); + + // Move item 456 close to, but not quite on the negative edge. + pi456.set_xyz(0x23, -0x7fff, 0x27); + LuaAssertStrEq(L, pm.outliers_debug_string("p"), "total:1 justout:0 wayout:0"); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,0,8" + "| L4:80,00,80" + "| L2:802,000,802" + "| L0:8023,0001,8027 456"); + + // Move item 456 so that it's on the negative edge, but not an outlier. + pi456.set_xyz(0x23, -0x7fff - 0.5, 0x27); + LuaAssertStrEq(L, pm.outliers_debug_string("p"), "total:1 justout:0 wayout:0"); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,0,8" + "| L4:80,00,80" + "| L2:802,000,802" + "| L0:8023,0000,8027 456"); + + // Move item 456 so that it's just barely a negative outlier. + pi456.set_xyz(0x23, -0x8000, 0x27); + LuaAssertStrEq(L, pm.outliers_debug_string("p"), "total:1 justout:1 wayout:0"); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,0,8" + "| L4:80,00,80" + "| L2:802,000,802" + "| L0:8023,0000,8027 456"); + + // Move item 456 so that it's significantly past the negative edge. + pi456.set_xyz(0x23, -0x8048, 0x27); + LuaAssertStrEq(L, pm.outliers_debug_string("p"), "total:1 justout:1 wayout:0"); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,0,8" + "| L4:80,00,80" + "| L2:802,000,802" + "| L0:8023,0000,8027 456"); + + // Move item 456 so that it's way past the negative edge. + pi456.set_xyz(0x23, -0x83048, 0x27); + LuaAssertStrEq(L, pm.outliers_debug_string("p"), "total:1 justout:0 wayout:1"); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:8,0,8" + "| L4:80,00,80" + "| L2:802,000,802" + "| L0:8023,0000,8027 456"); + + // Test the calculation of search bboxes. + // The two corners are deliberately not in low-high order. + scan.clear(); + scan.set_plane("p"); + scan.set_bbox_given_center_radius(util::XYZ(0x23, 0x97, 0x103), 2.0f); + LuaAssertStrEq(L, pm.search_bboxes_debug_string(scan), + "|Level 8 0,0,0 - 0,0,0" + "|Level 6 8,8,8 - 8,8,8" + "|Level 4 80,80,81 - 80,80,81" + "|Level 2 802,809,810 - 802,809,810" + "|Level 0 8021,8095,8101 - 8025,8099,8105"); + + // TESTS OF SCANNING + + // Store a single object in the map to scan. + pm.untrack_all(); + pi123.set_pos("p", 0x12, 0x34, 0x45); + pi123.track(&pm); + + // Set up a scan with a radius of zero, centered + // right on the one object. Check the bboxes to make + // sure they only include the one cell. + scan.clear(); + scan.set_plane("p"); + scan.set_bbox_given_center_radius(util::XYZ(0x12, 0x34, 0x45), 0.0); + LuaAssertStrEq(L, pm.search_bboxes_debug_string(scan), + "|Level 8 0,0,0 - 0,0,0" + "|Level 6 8,8,8 - 8,8,8" + "|Level 4 80,80,80 - 80,80,80" + "|Level 2 801,803,804 - 801,803,804" + "|Level 0 8012,8034,8045 - 8012,8034,8045"); + + // Run the scan with radius zero. It should find the one object. + LuaAssertStrEq(L, pm.scan_steps_debug_string(scan), + "|L8:root" + "| L7:2,2,2" + "| L6:8,8,8" + "| L5:20,20,20" + "| L4:80,80,80" + "| L3:200,200,201" + "| L2:801,803,804" + "| L1:2004,200d,2011" + "| L0:8012,8034,8045 123" + "|Result: 123"); + + // Bump the scan over a half-unit. It should have the same + // bboxes as before, since a half-unit isn't enough to shift + // from one cell to the next. + scan.clear(); + scan.set_plane("p"); + scan.set_bbox_given_center_radius(util::XYZ(0x12 + 0.5, 0x34, 0x45), 0.0); + LuaAssertStrEq(L, pm.search_bboxes_debug_string(scan), + "|Level 8 0,0,0 - 0,0,0" + "|Level 6 8,8,8 - 8,8,8" + "|Level 4 80,80,80 - 80,80,80" + "|Level 2 801,803,804 - 801,803,804" + "|Level 0 8012,8034,8045 - 8012,8034,8045"); + + // Run the scan with radius zero, but one half-step away + // from the object. It should encounter the cell containing the + // one object, but the object should get removed in the final + // filtering step. + LuaAssertStrEq(L, pm.scan_steps_debug_string(scan), + "|L8:root" + "| L7:2,2,2" + "| L6:8,8,8" + "| L5:20,20,20" + "| L4:80,80,80" + "| L3:200,200,201" + "| L2:801,803,804" + "| L1:2004,200d,2011" + "| L0:8012,8034,8045 123" + "|Result: "); + + // Next, expand the scan radius to huge. Examine the bboxes + // to make sure they cover the entire PlaneTree. + scan.clear(); + scan.set_plane("p"); + scan.set_bbox_given_center_radius(util::XYZ(0x12, 0x34, 0x45), 100000.0); + LuaAssertStrEq(L, pm.search_bboxes_debug_string(scan), + "|Level 8 0,0,0 - 0,0,0" + "|Level 6 0,0,0 - f,f,f" + "|Level 4 00,00,00 - ff,ff,ff" + "|Level 2 000,000,000 - fff,fff,fff" + "|Level 0 0000,0000,0000 - ffff,ffff,ffff"); + + // Walk the tree using the expansive search bboxes. It should + // find the one object, and it should still only traverse the same + // cells, because those are the only cells that exist. + LuaAssertStrEq(L, pm.scan_steps_debug_string(scan), + "|L8:root" + "| L7:2,2,2" + "| L6:8,8,8" + "| L5:20,20,20" + "| L4:80,80,80" + "| L3:200,200,201" + "| L2:801,803,804" + "| L1:2004,200d,2011" + "| L0:8012,8034,8045 123" + "|Result: 123"); + + // Add another object to the tree. Then, scan again + // using the expansive search. It should find both objects. + pi456.set_pos("p", 0x14, 0x35, 0x30); + pi456.track(&pm); + LuaAssertStrEq(L, pm.scan_steps_debug_string(scan), + "|L8:root" + "| L7:2,2,2" + "| L6:8,8,8" + "| L5:20,20,20" + "| L4:80,80,80" + "| L3:200,200,200" + "| L2:801,803,803" + "| L1:2005,200d,200c" + "| L0:8014,8035,8030 456" + "| L3:200,200,201" + "| L2:801,803,804" + "| L1:2004,200d,2011" + "| L0:8012,8034,8045 123" + "|Result: 123,456"); + + // We're going to test that sphere of radius 0.0 works. This is an important + // test because the sphere calculation involves calculating the inverse of + // the radius, which has to be special-cased when radius is zero. We need + // special-case code for this. In this test case, we set up a scan with two + // objects in the same cell, just a smidge apart. Use a sphere scan with + // radius zero, centered right on object 123 (but missing object 456). + pm.untrack_all(); + pi123.set_pos("p", 0x12 + 0.1, 0x34, 0x45); + pi456.set_pos("p", 0x12 + 0.2, 0x34, 0x45); + pi123.track(&pm); + pi456.track(&pm); + scan.clear(); + scan.set_plane("p"); + scan.set_shape(PlaneScan::SPHERE); + scan.set_bbox_given_center_radius(util::XYZ(0x12 + 0.1, 0x34, 0x45), 0.0); + LuaAssertStrEq(L, pm.scan_steps_debug_string(scan), + "|L8:root" + "| L7:2,2,2" + "| L6:8,8,8" + "| L5:20,20,20" + "| L4:80,80,80" + "| L3:200,200,201" + "| L2:801,803,804" + "| L1:2004,200d,2011" + "| L0:8012,8034,8045 123,456" + "|Result: 123"); + + // We're going to test that 'whole plane' searches work. + // These use an infinite radius. + pm.untrack_all(); + pi123.set_pos("p", 0x12, 0x34, 0x45); + pi123.track(&pm); + scan.clear(); + scan.set_plane("p"); + scan.set_whole_plane(); + LuaAssertStrEq(L, pm.search_bboxes_debug_string(scan), + "|Level 8 0,0,0 - 0,0,0" + "|Level 6 0,0,0 - f,f,f" + "|Level 4 00,00,00 - ff,ff,ff" + "|Level 2 000,000,000 - fff,fff,fff" + "|Level 0 0000,0000,0000 - ffff,ffff,ffff"); + LuaAssertStrEq(L, pm.scan_steps_debug_string(scan), + "|L8:root" + "| L7:2,2,2" + "| L6:8,8,8" + "| L5:20,20,20" + "| L4:80,80,80" + "| L3:200,200,201" + "| L2:801,803,804" + "| L1:2004,200d,2011" + "| L0:8012,8034,8045 123" + "|Result: 123"); + + // Set up a tree with a single object that's outside + // the tree's bounding box (an outlier). Print the tree + // to verify that the object ended up on the edge. + pm.untrack_all(); + pi123.set_pos("p", 0x100000, 0x16, 0x23); + pi123.track(&pm); + LuaAssertStrEq(L, pm.tree_debug_string("p"), + "|L8:root" + "| L6:f,8,8" + "| L4:ff,80,80" + "| L2:fff,801,802" + "| L0:ffff,8016,8023 123"); + + // Now set up a scan around the target outlier. Print out + // the scan bboxes. It should be scanning the edge cell that + // contains the outlier. It also contains a few other cells + // because the radius is nonzero. + scan.clear(); + scan.set_plane("p"); + scan.set_bbox_given_center_radius(util::XYZ(0x100000, 0x16, 0x23), 0.2); + LuaAssertStrEq(L, pm.search_bboxes_debug_string(scan), + "|Level 8 0,0,0 - 0,0,0" + "|Level 6 f,8,8 - f,8,8" + "|Level 4 ff,80,80 - ff,80,80" + "|Level 2 fff,801,802 - fff,801,802" + "|Level 0 ffff,8015,8022 - ffff,8016,8023"); + + // Confirm that the scan finds the outlier. + LuaAssertStrEq(L, pm.scan_steps_debug_string(scan), + "|L8:root" + "| L7:3,2,2" + "| L6:f,8,8" + "| L5:3f,20,20" + "| L4:ff,80,80" + "| L3:3ff,200,200" + "| L2:fff,801,802" + "| L1:3fff,2005,2008" + "| L0:ffff,8016,8023 123" + "|Result: 123"); return 0; } diff --git a/luprex/core/cpp/planemap.hpp b/luprex/core/cpp/planemap.hpp index bc7a71d6..ffdbd741 100644 --- a/luprex/core/cpp/planemap.hpp +++ b/luprex/core/cpp/planemap.hpp @@ -76,21 +76,110 @@ #include "wrap-vector.hpp" #include "wrap-map.hpp" #include "wrap-string.hpp" - +#include "wrap-bytell-hash-map.hpp" #include "util.hpp" +#include "luastack.hpp" #include +#include +#include class PlaneMap; +class PlaneTree; + +class PlaneScan : public eng::nevernew { +public: + friend class PlaneMap; + friend class PlaneTree; + enum Shape { BOX, CYLINDER, SPHERE }; +private: + // The plane to scan. + eng::string plane_; + + // The bounding box of the scan. + util::XYZ center_, radius_; + + // If you scan a cylinder or SPHERE, it actually + // scans the bounding box first, then clips out + // the parts that aren't correct. + Shape shape_; + + // When true, the items are sorted by ID. + // WARNING: setting this to false can create + // nondeterminism. Scans by lua should always be sorted. + bool sorted_; + + // The near ID, if nonzero, is either PREPENDED to the + // results, or OMITTED from the results, depending on include_near_. + int64_t near_; + bool include_near_; + + // If this is true, items on the nowhere plane are not scanned. + bool omit_nowhere_; + +public: + void clear() { + plane_ = ""; + shape_ = BOX; + sorted_ = true; + near_ = 0; + include_near_ = false; + omit_nowhere_ = false; + radius_ = center_ = util::XYZ(); + } + PlaneScan() { clear(); } + + // Convert a lua table into a scan configuration. + // + // If there is an error in the configuration, returns an + // error message, otherwise returns empty string. + // + // Caution: if this detects the configuration flag 'near', + // then it stores the near ID. However, it doesn't fetch + // the center and plane. It is the caller's responsibility + // to check if 'near' has been set, and if so, store a center + // and plane. This is admittedly ugly, but it eliminates + // a dependency on the world module. + // + eng::string configure(const LuaStack &LS, LuaSlot slot); + + void set_bbox_given_center_radius(const util::XYZ ¢er, float r) { + set_center(center); + set_radius(r); + } + + void set_whole_plane() { + shape_ = BOX; + center_ = util::XYZ(); + radius_.x = std::numeric_limits::infinity(); + radius_.y = radius_.z = radius_.x; + } + + void set_center(const util::XYZ ¢er) { center_ = center; } + void set_radius(const util::XYZ &radius) { radius_ = radius; } + void set_radius(float f) { radius_.x = radius_.y = radius_.z = f; } + void set_plane(std::string_view p) { plane_ = p; } + void set_shape(Shape s) { shape_ = s; } + void set_sorted(bool s) { sorted_ = s; } + void set_near(int64_t id, bool inc) { near_ = id; include_near_ = inc; } + void set_omit_nowhere(bool b) { omit_nowhere_ = b; } + + int64_t near() const { return near_; } + + // Reveal the contents of the PlaneScan as a string. + eng::string debug_string() const; +}; class PlaneItem : public eng::nevernew { - friend class PlaneMap; - + friend class PlaneTree; + friend class PlaneMap; private: - PlaneMap *pmap_; + PlaneTree *tree_; + PlaneItem *prev_; + PlaneItem *next_; + int64_t id_; eng::string plane_; float x_, y_, z_; - int64_t id_; public: PlaneItem(); @@ -105,43 +194,51 @@ public: const float y() const { return y_; } const float z() const { return z_; } - void untrack(); void track(PlaneMap *pmap); + void untrack() { track(nullptr); } void set_pos(const eng::string &plane, float x, float y, float z); - void set_xyz(float x, float y, float z) { set_pos(plane_, x, y, z); } + void set_xyz(float x, float y, float z); }; class PlaneMap : public eng::nevernew { friend class PlaneItem; -private: - using EltVec = eng::vector; - using Plane = eng::map; - eng::map planes_; - void remove(const eng::string &plane, int64_t cell, PlaneItem *client); - void insert(const eng::string &plane, int64_t cell, PlaneItem *client); - + friend class PlaneTree; public: using IdVector = util::IdVector; + +private: + float default_radius_; + eng::map> planes_; + +public: + // No special code is needed for construction or destruction. PlaneMap(); ~PlaneMap(); - // Caution: scan_radius is not deterministically ordered unless sort=true. - // - // exclude_nowhere - if true, and plane="nowhere", nothing is scanned. - // special - an ID that is considered special. If zero, there is no special ID. - // omit - if true, remove the special ID from the list. - // if false, prepend the special ID to the head of the list. - // - IdVector scan_radius(const eng::string &plane, float x, float y, float radius, bool exclude_nowhere, int64_t special, bool omit) const; - IdVector scan_radius_unsorted(const eng::string &plane, float x, float y, float radius, bool exclude_nowhere, int64_t special, bool omit) const; + // The 'insert' and 'remove' operators are inside class + // PlaneItem. See PlaneItem::track and PlaneItem::untrack. + // Scan the PlaneMap for items, return their IDs. + IdVector scan(const PlaneScan &s) const; + + // Set the default radius for all planes. + // Maybe we'll make it adaptive some day. + void set_default_radius(float r) { default_radius_ = r; } + + // Return a debug string for the specified plane. + // This is for unit testing. + eng::string tree_debug_string(const eng::string &plane); + eng::string outliers_debug_string(const eng::string &plane); + eng::string search_bboxes_debug_string(const PlaneScan &scan); + eng::string scan_steps_debug_string(const PlaneScan &scan); + + // Untrack all planeitems. This is for unit testing. + void untrack_all(); + private: // unit testing stuff. friend int lfn_unittests_planemap(lua_State *L); - EltVec get_cell(const eng::string &plane, int64_t cell) const; - int total_cells() const; - void clear() { planes_.clear(); } }; diff --git a/luprex/core/cpp/pprint.cpp b/luprex/core/cpp/pprint.cpp index eada8c22..61a2c918 100644 --- a/luprex/core/cpp/pprint.cpp +++ b/luprex/core/cpp/pprint.cpp @@ -5,6 +5,7 @@ #include "table.hpp" #include +#include void atomic_print(LuaStack &LS, LuaSlot val, bool quote, std::ostream *os) { @@ -23,11 +24,15 @@ void atomic_print(LuaStack &LS, LuaSlot val, bool quote, std::ostream *os) { return; case LUA_TNUMBER: { double value = LS.cknumber(val); - int64_t ivalue = int64_t(value); - if (double(ivalue) == value) { - (*os) << ivalue; + if (std::isnan(value)) { + (*os) << "nan"; } else { - (*os) << value; + int64_t ivalue = int64_t(value); + if (double(ivalue) == value) { + (*os) << ivalue; + } else { + (*os) << value; + } } return; } @@ -38,6 +43,11 @@ void atomic_print(LuaStack &LS, LuaSlot val, bool quote, std::ostream *os) { (*os) << ""; return; } + case LUA_TLIGHTUSERDATA: { + LuaToken token = LS.cktoken(val); + (*os) << "[" << token.str() << "]"; + return; + } default: (*os) << "<" << lua_typename(LS.state(), tt) << ">"; return; diff --git a/luprex/core/cpp/source.cpp b/luprex/core/cpp/source.cpp index 9ef21c4a..ccb9142e 100644 --- a/luprex/core/cpp/source.cpp +++ b/luprex/core/cpp/source.cpp @@ -56,8 +56,7 @@ LuaDefine(classname, "classtable", "get the class name from a class table") { return LS.result(); } -static void get_reg_name(const LuaFunctionReg *reg, std::string_view &classname, std::string_view &funcname) { - std::string_view name(reg->get_name()); +static void get_reg_name(std::string_view name, std::string_view &classname, std::string_view &funcname) { size_t upos = name.find('_'); if (upos == std::string_view::npos) { funcname = name; @@ -280,7 +279,7 @@ static void source_load_cfunctions(lua_State *L) { if ((func != nullptr) && (!r->get_sandbox())) { std::string_view classname; std::string_view funcname; - get_reg_name(r, classname, funcname); + get_reg_name(r->get_name(), classname, funcname); if (classname.empty()) { LS.getglobaltable(classobj); LS.rawset(classobj, funcname, func); @@ -293,6 +292,31 @@ static void source_load_cfunctions(lua_State *L) { LS.result(); } +// Load all the 'LuaConstant' constants into the lua state. +// +static void source_load_cconstants(lua_State *L) { + LuaVar classobj, value; + LuaStack LS(L, classobj, value); + for (auto r = LuaConstantReg::All; r != nullptr; r=r->next()) { + if (r->get_tokenvalue().empty()) { + LS.set(value, r->get_numbervalue()); + } else { + LS.set(value, r->get_tokenvalue()); + } + std::string_view classname; + std::string_view funcname; + get_reg_name(r->get_name(), classname, funcname); + if (classname.empty()) { + LS.getglobaltable(classobj); + LS.rawset(classobj, funcname, value); + } else { + LS.makeclass(classobj, classname); + LS.rawset(classobj, funcname, value); + } + } + LS.result(); +} + // Run all the closures from the source database. // static eng::string source_load_lfunctions(lua_State *L) { @@ -341,18 +365,12 @@ static eng::string source_load_lfunctions(lua_State *L) { eng::string SourceDB::rebuild() { lua_State *L = lua_state_; - LuaVar mathclass; - LuaStack LS(L, mathclass); + LuaVar mathclass, httpclass, jsonnull; + LuaStack LS(L, mathclass, httpclass, jsonnull); source_clear_globals(L); source_load_cfunctions(L); + source_load_cconstants(L); eng::string errs = source_load_lfunctions(L); - - // A few builtin constants. These are hardwired. - LS.makeclass(mathclass, "math"); - LS.rawset(mathclass, "pi", M_PI); - LS.rawset(mathclass, "huge", HUGE_VAL); - LS.rawset(mathclass, "maxint", LuaStack::MAXINT); - LS.result(); return errs; } @@ -466,7 +484,7 @@ void SourceDB::register_lua_builtins() { for (auto reg = LuaFunctionReg::All; reg != nullptr; reg=reg->next()) { std::string_view funcname; std::string_view classname; - get_reg_name(reg, classname, funcname); + get_reg_name(reg->get_name(), classname, funcname); if (classname.empty()) { LS.getglobaltable(classtab); } else { @@ -524,7 +542,7 @@ eng::string SourceDB::function_docs(const LuaStack &LS0, LuaSlot fn) { } std::string_view classname; std::string_view funcname; - get_reg_name(reg, classname, funcname); + get_reg_name(reg->get_name(), classname, funcname); eng::ostringstream oss; util::StringVec docs = util::split_docstring(reg->get_docs()); oss << "function "; @@ -747,6 +765,11 @@ LuaDefineBuiltin(math_sqrt, "x", "return the square root of x"); LuaDefineBuiltin(math_tan, "x", "return the tangent of x in radians"); LuaDefineBuiltin(math_tanh, "x", "return the hyperbolic tangent of x in radians"); LuaSandboxBuiltin(math_log10, "", ""); +LuaNumberConstant(math_pi, M_PI, ""); +LuaNumberConstant(math_huge, HUGE_VAL, ""); +LuaNumberConstant(math_nan, NAN, ""); +LuaNumberConstant(math_maxint, LuaStack::MAXINT, ""); + // math.random and math.randomseed are in world-accessor.cpp, because // generating random numbers must manipulate global state which is // stored in the world model. diff --git a/luprex/core/cpp/util.cpp b/luprex/core/cpp/util.cpp index dd3c2ecd..5b942bd1 100644 --- a/luprex/core/cpp/util.cpp +++ b/luprex/core/cpp/util.cpp @@ -65,8 +65,10 @@ bool valid_double(string_view value) { int64_t to_int64(string_view value, int64_t errval) { int64_t result; - const char *last = value.data() + value.size(); - auto r = std::from_chars(value.data(), last, result, 10); + const char *p = value.data(); + const char *last = p + value.size(); + if ((p < last) && (*p == '+')) p++; + auto r = std::from_chars(p, last, result, 10); if (r.ec != std::errc()) return errval; if (r.ptr != last) return errval; return result; @@ -74,6 +76,7 @@ int64_t to_int64(string_view value, int64_t errval) { uint64_t to_hex64(string_view value, uint64_t errval) { uint64_t result; + if (sv::zfront(value) == '-') return errval; const char *last = value.data() + value.size(); auto r = std::from_chars(value.data(), last, result, 16); if (r.ec != std::errc()) return errval; @@ -204,6 +207,15 @@ string_view read_to_line(string_view &source) { return result; } +bool read_prefix(string_view &source, string_view prefix) { + if (0 == source.compare(0, prefix.size(), prefix)) { + source.remove_prefix(prefix.size()); + return true; + } else { + return false; + } +} + string_view read_to_space(string_view &source) { size_t pos1 = 0; while ((pos1 < source.size()) && (!ascii_isspace(source[pos1]))) { @@ -243,57 +255,119 @@ string_view read_ascii_identifier(string_view &source) { return result; } +std::string_view read_number(string_view &source, bool plus, bool minus, bool dec, bool exp) { + const char *p = source.data(); + const char *l = p + source.size(); + if (p == l) return source.substr(0, 0); + char sign = *p; + if (sign == '+') { + if (!plus) return source.substr(0, 0); + p++; + } + if (sign == '-') { + if (!minus) return source.substr(0, 0); + p++; + } + if (p == l) return source.substr(0, 0); + bool have_digits = false; + while ((p < l) && (ascii_isdigit(*p))) { + have_digits = true; + p++; + } + if ((p < l) && dec && (*p == '.')) { + p++; + while ((p < l) && (ascii_isdigit(*p))) { + have_digits = true; + p++; + } + } + if (!have_digits) return source.substr(0, 0); + if ((p < l) && exp && ((*p == 'e')||(*p == 'E'))) { + p++; + if ((p < l) && ((*p == '+') || (*p == '-'))) { + p++; + } + bool have_exp = false; + while ((p < l) && (ascii_isdigit(*p))) { + have_exp = true; + p++; + } + if (!have_exp) return source.substr(0, 0); + } + string_view result = source.substr(0, p - source.data()); + source.remove_prefix(result.size()); + return result; +} + +int32_t read_ascii_char(string_view &source) { + if (source.empty()) return -1; + int32_t result = source.front(); + source.remove_prefix(1); + return result; +} + +int32_t read_codepoint_utf8(string_view &source) { + size_t size = source.size(); + if (size == 0) return -1; + const unsigned char *bytes = (const unsigned char *)source.data(); + int codepoint; + size_t seqlen; + if ((bytes[0] & 0x80) == 0x00) { + // U+0000 to U+007F + codepoint = (bytes[0] & 0x7F); + seqlen = 1; + } else if ((bytes[0] & 0xE0) == 0xC0) { + // U+0080 to U+07FF + codepoint = (bytes[0] & 0x1F); + seqlen = 2; + } else if ((bytes[0] & 0xF0) == 0xE0) { + // U+0800 to U+FFFF + codepoint = (bytes[0] & 0x0F); + seqlen = 3; + } else if ((bytes[0] & 0xF8) == 0xF0) { + // U+10000 to U+10FFFF + codepoint = (bytes[0] & 0x07); + seqlen = 4; + } else { + return -1; + } + + if (seqlen > size) { + return -1; + } + + for (size_t i = 1; i < seqlen; ++i) { + if ((bytes[i] & 0xC0) != 0x80) return -1; + codepoint = (codepoint << 6) | (bytes[i] & 0x3F); + } + + if ((codepoint > 0x10FFFF) || + ((codepoint >= 0xD800) && (codepoint <= 0xDFFF)) || + ((codepoint <= 0x007F) && (seqlen != 1)) || + ((codepoint >= 0x0080) && (codepoint <= 0x07FF) && (seqlen != 2)) || + ((codepoint >= 0x0800) && (codepoint <= 0xFFFF) && (seqlen != 3)) || + ((codepoint >= 0x10000) && (codepoint <= 0x1FFFFF) && (seqlen != 4))) { + return -1; + } + + source.remove_prefix(seqlen); + return codepoint; +} + bool valid_utf8(string_view s) { - const unsigned char *bytes = (const unsigned char *)s.data(); - const unsigned char *tail = bytes + s.size(); - unsigned int codepoint; - int seqlen; - - while (bytes < tail) { - if ((bytes[0] & 0x80) == 0x00) { - // U+0000 to U+007F - codepoint = (bytes[0] & 0x7F); - seqlen = 1; - } else if ((bytes[0] & 0xE0) == 0xC0) { - // U+0080 to U+07FF - codepoint = (bytes[0] & 0x1F); - seqlen = 2; - } else if ((bytes[0] & 0xF0) == 0xE0) { - // U+0800 to U+FFFF - codepoint = (bytes[0] & 0x0F); - seqlen = 3; - } else if ((bytes[0] & 0xF8) == 0xF0) { - // U+10000 to U+10FFFF - codepoint = (bytes[0] & 0x07); - seqlen = 4; - } else { - return false; - } - - if (bytes + seqlen > tail) { - return false; - } - - for (int i = 1; i < seqlen; ++i) { - if ((bytes[i] & 0xC0) != 0x80) return false; - codepoint = (codepoint << 6) | (bytes[i] & 0x3F); - } - - if ((codepoint > 0x10FFFF) || - ((codepoint >= 0xD800) && (codepoint <= 0xDFFF)) || - ((codepoint <= 0x007F) && (seqlen != 1)) || - ((codepoint >= 0x0080) && (codepoint <= 0x07FF) && (seqlen != 2)) || - ((codepoint >= 0x0800) && (codepoint <= 0xFFFF) && (seqlen != 3)) || - ((codepoint >= 0x10000) && (codepoint <= 0x1FFFFF) && (seqlen != 4))) { - return false; - } - - bytes += seqlen; + while (!s.empty()) { + int32_t codepoint = read_codepoint_utf8(s); + if (codepoint < 0) return false; } return true; } +bool valid_number(string_view s, bool plus, bool minus, bool dec, bool exp) { + read_number(s, plus, minus, dec, exp); + return s.empty(); +} + } // namespace sv @@ -334,6 +408,8 @@ void quote_string(const eng::string &s, std::ostream *os) { (*os) << (usesinglequote ? "\"" : "\\\""); } else if (c == '\'') { (*os) << (usesinglequote ? "\\'" : "'"); + } else if (c == '\\') { + (*os) << "\\\\"; } else { (*os) << c; } @@ -344,7 +420,7 @@ void quote_string(const eng::string &s, std::ostream *os) { case '\t': (*os) << "\\t"; break; case '\r': (*os) << "\\r"; break; default: - (*os) << "\\" << std::setfill('0') << std::setw(3) << value; + (*os) << "\\" << dec.width(3).fill('0').val(value); break; } } @@ -352,6 +428,52 @@ void quote_string(const eng::string &s, std::ostream *os) { (*os) << (usesinglequote ? '\'' : '"'); } +void base64_encode(std::string_view str, std::ostream *oss) { + const char *encode_tab = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + const char *s = str.data(); + size_t size = str.size(); + for (size_t i = 0; i < size; i += 3) { + uint32_t block = ((unsigned char)(s[i])) << 16; + if (i + 1 < size) block |= ((unsigned char)(s[i + 1])) << 8; + if (i + 2 < size) block |= ((unsigned char)(s[i + 2])); + (*oss) << encode_tab[(block>>18)&0x3F]; + (*oss) << encode_tab[(block>>12)&0x3F]; + (*oss) << ((i + 1 < size) ? encode_tab[(block>>6)&0x3F] : '='); + (*oss) << ((i + 2 < size) ? encode_tab[(block>>0)&0x3F] : '='); + } +} + +bool base64_decode(std::string_view str, std::ostream *oss) { + uint32_t chunk = 0; + int fill = 0; + int skip = 0; + bool clean = true; + for (int i = 0; i < int(str.size()); i++) { + char c = str[i]; + uint32_t value; + + if ((c >= 'A') && (c <= 'Z')) value = c - 'A'; + else if ((c >= 'a') && (c <= 'z')) value = c - 'a' + 26; + else if ((c >= '0') && (c <= '9')) value = c - '0' + 52; + else if (c == '+') value = 62; + else if (c == '/') value = 63; + else if (c == '=') { value = 0; skip ++; } + else { clean=false; continue; } + + chunk = (chunk << 6) | value; + fill ++; + if (fill == 4) { + oss->put((chunk>>16) & 0xFF); + if (skip < 2) oss->put((chunk>>8) & 0xFF); + if (skip < 1) oss->put(chunk & 0xFF); + chunk = 0; fill = 0; skip = 0; + } + } + if (fill != 0) clean = false; + return clean; +} + IdVector id_vector_create(int64_t id1, int64_t id2, int64_t id3, int64_t id4) { IdVector result; if (id1 >= 0) result.push_back(id1); @@ -361,14 +483,18 @@ IdVector id_vector_create(int64_t id1, int64_t id2, int64_t id3, int64_t id4) { return result; } -eng::string id_vector_debug_string(const IdVector &idv) { - eng::ostringstream oss; +void print_id_vector(const IdVector &idv, std::ostream *os) { bool first = true; for (int64_t id : idv) { - if (!first) oss << ","; - oss << id; + if (!first) (*os) << ","; + (*os) << id; first = false; } +} + +eng::string id_vector_debug_string(const IdVector &idv) { + eng::ostringstream oss; + print_id_vector(idv, &oss); return oss.str(); } @@ -406,8 +532,7 @@ HashValue hash_id_vector(const IdVector &idv) { eng::string hash_to_hex(const HashValue &hv) { eng::ostringstream oss; - oss << std::hex << std::setw(16) << std::setfill('0') << hv.first; - oss << std::hex << std::setw(16) << std::setfill('0') << hv.second; + oss << hex64.val(hv.first) << hex64.val(hv.second); return oss.str(); } static inline uint64_t Rot64(uint64_t x, int k) @@ -530,6 +655,52 @@ eng::string toupper(eng::string input) { return input; } +static void buffer_codepoint_utf8(int32_t scp, char *buffer) { + uint32_t cp = (uint32_t)scp; + unsigned char *c = (unsigned char *)buffer; + if (cp <= 0x7F) { + c[0] = cp; + c[1] = 0; + } + else if (cp <= 0x7FF) { + c[0] = (cp>>6)+192; + c[1] = (cp&63)+128; + c[2] = 0; + } + else if (cp <= 0xFFFF) { + if (0xd800 <= cp && cp <= 0xdfff) { + c[0] = 0; + } else { + c[0] = (cp>>12)+224; + c[1] = ((cp>>6)&63)+128; + c[2] = (cp&63)+128; + c[3] = 0; + } + } + else if (cp <= 0x10FFFF) { + c[0] = (cp>>18)+240; + c[1] = ((cp>>12)&63)+128; + c[2] = ((cp>>6)&63)+128; + c[3] = (cp&63)+128; + c[4] = 0; + } else { + c[0] = 0; + } +} + +eng::string get_codepoint_utf8(uint32_t cp) { + char buffer[5]; + buffer_codepoint_utf8(cp, buffer); + return eng::string(buffer); +} + +bool write_codepoint_utf8(int32_t cp, std::ostream *s) { + char buffer[5]; + buffer_codepoint_utf8(cp, buffer); + (*s) << buffer; + return buffer[0] != 0; +} + double distance_squared(double x1, double y1, double x2, double y2) { double dx = x1 - x2; double dy = y1 - y2; @@ -549,35 +720,20 @@ eng::string XYZ::debug_string() const { return oss.str(); } - } // namespace util -std::ostream &operator<<(std::ostream &oss, const util::hex64 &v) { - oss << "0x" << std::setw(16) << std::setfill('0') << std::hex; - return oss; -} -std::ostream &operator<<(std::ostream &oss, const util::hex32 &v) { - oss << "0x" << std::setw(8) << std::setfill('0') << std::hex; - return oss; -} - -std::ostream &operator<<(std::ostream &oss, const util::hex16 &v) { - oss << "0x" << std::setw(4) << std::setfill('0') << std::hex; - return oss; -} - -std::ostream &operator<<(std::ostream &oss, const util::hex8 &v) { - oss << "0x" << std::setw(2) << std::setfill('0') << std::hex; - return oss; +static std::string_view read_number_x(const char *p, bool plus, bool minus, bool dec, bool exp) { + std::string_view source = p; + return sv::read_number(source, plus, minus, dec, exp); } LuaDefine(unittests_util, "", "some unit tests") { // test str_to_int64, str_to_double LuaAssert(L, sv::to_int64("123") == 123); - LuaAssert(L, sv::to_int64("123.4") == INT64_MIN); - LuaAssert(L, sv::to_int64("12ab") == INT64_MIN); - LuaAssert(L, sv::to_int64("") == INT64_MIN); + LuaAssert(L, sv::to_int64("123.4") == INT64_MAX); + LuaAssert(L, sv::to_int64("12ab") == INT64_MAX); + LuaAssert(L, sv::to_int64("") == INT64_MAX); LuaAssert(L, sv::to_double("123.5") == 123.5); LuaAssert(L, std::isnan(sv::to_double("12ab"))); LuaAssert(L, std::isnan(sv::to_double(""))); @@ -689,6 +845,20 @@ LuaDefine(unittests_util, "", "some unit tests") { LuaAssert(L, util::hash_to_double(0x1000000000000000) == 1.0/16.0); LuaAssert(L, util::hash_to_double(0x7000000000000000) == 7.0/16.0); LuaAssert(L, util::hash_to_double(0xF000000000000000) == 15.0/16.0); + + // Test read_number allowing everything. + LuaAssert(L, read_number_x("123x", true, true, true, true) == "123"); + LuaAssert(L, read_number_x("123.3x", true, true, true, true) == "123.3"); + LuaAssert(L, read_number_x("123.x", true, true, true, true) == "123."); + LuaAssert(L, read_number_x("123..x", true, true, true, true) == "123."); + LuaAssert(L, read_number_x("-123x", true, true, true, true) == "-123"); + LuaAssert(L, read_number_x("+123x", true, true, true, true) == "+123"); + LuaAssert(L, read_number_x("+-123x", true, true, true, true) == ""); + LuaAssert(L, read_number_x("-123.02e05x", true, true, true, true) == "-123.02e05"); + LuaAssert(L, read_number_x("-123e-5x", true, true, true, true) == "-123e-5"); + LuaAssert(L, read_number_x("-123e+5x", true, true, true, true) == "-123e+5"); + LuaAssert(L, read_number_x("-123e+x", true, true, true, true) == ""); + return 0; } diff --git a/luprex/core/cpp/util.hpp b/luprex/core/cpp/util.hpp index 7c60d894..4b515005 100644 --- a/luprex/core/cpp/util.hpp +++ b/luprex/core/cpp/util.hpp @@ -26,7 +26,13 @@ #include #include #include +<<<<<<< HEAD #include +======= +#include +#include +#include +>>>>>>> 4e754377ee4922940c37a20710314befeff0d84e #include "luastack.hpp" #include "spookyv2.hpp" @@ -60,9 +66,17 @@ bool valid_double(string_view v); bool valid_int64(string_view v); bool valid_hex64(string_view v); -// Parse numbers as int32, int64, or double. Returns errval on failure. +// Convert strings to numbers. Returns errval on failure. +// +// The integer parser accepts a sequence of digits, +// with or without a + or - sign. The hex parser +// does not allow a + or - sign. For both the int64 +// and hex64 parser, it is a failure if the number +// does not fit in 64 bits. The double parser does +// not accept the strings 'nan' or 'inf'. +// double to_double(string_view v, double errval = std::numeric_limits::quiet_NaN()); -int64_t to_int64(string_view v, int64_t errval = std::numeric_limits::min()); +int64_t to_int64(string_view v, int64_t errval = std::numeric_limits::max()); uint64_t to_hex64(string_view v, uint64_t errval = std::numeric_limits::max()); // Trim whitspace from a string_view. @@ -116,6 +130,13 @@ string_view read_to_sep(string_view &source, char sep); // string_view read_to_line(string_view &source); +// Read a prefix string from a string_view. +// +// Returns false if the string view doesn't start with +// the specified prefix. +// +bool read_prefix(string_view &source, string_view prefix); + // Read from a string_view until whitespace is reached. // // If there's any whitespace in the source, returns the text @@ -137,9 +158,46 @@ string_view read_nbytes(string_view &source, int nbytes); // string_view read_ascii_identifier(string_view &source); +// Read a number from a string view +// +// This is basically a regex pattern matching routine +// hardwired with the regex for numbers. You must +// specify which of the following parts of the regex +// are allowed or not: +// +// * plus sign +// * minus sign +// * decimal point +// * scientific notation exponents +// +// Returns the number as a string_view. There is +// no guarantee that the number is small enough to +// fit into any particular number of bits. This +// always uses base 10. +// +std::string_view read_number(string_view &source, bool plus, bool minus, bool dec, bool exp); + +// Read an ascii character from a string. +// +// Returns -1 if the string is empty. +// +int32_t read_ascii_char(string_view &source); + +// Read a UTF8 codepoint from a string_view. +// +// If the next thing in the string_view isn't a valid +// codepoint, returns -1 and doesn't update the view. +// +int32_t read_codepoint_utf8(string_view &source); + // Return true if the string is valid utf-8. bool valid_utf8(string_view s); +// Return true if the number conforms to the spec. +// See read_number for more information. +// +bool valid_number(string_view v, bool plus, bool minus, bool dec, bool exp); + } // namespace sv namespace util { @@ -176,9 +234,22 @@ double profiling_clock(); // Output a string to a stream using Lua string escaping and quoting. void quote_string(const eng::string &str, std::ostream *os); +// base64 encode. +void base64_encode(std::string_view v, std::ostream *oss); + +// base64 decode. +// +// Returns true if the base64 was 'clean' base64, as +// opposed to base64 with extraneous characters. +// +bool base64_decode(std::string_view v, std::ostream *oss); + // ID vector quick create. IdVector id_vector_create(int64_t id1=-1, int64_t id2=-1, int64_t id3=-1, int64_t id4=-1); +// Print an ID vector to a stream. +void print_id_vector(const IdVector &idv, std::ostream *os); + // ID vector debug string. eng::string id_vector_debug_string(const IdVector &idv); @@ -198,6 +269,16 @@ eng::string hash_to_hex(const HashValue &hash); // This is a good hash, but not cryptographically good. uint64_t hash_ints(uint64_t n1, uint64_t n2, uint64_t n3, uint64_t n4); +// Hash a single 64-bit integer. +// This is a good hash, but not cryptographically good. +// Published by David Stafford in his article 'Better Bit Mixing'. +inline uint64_t hash_int(uint64_t x) { + x = (x ^ (x >> 30)) * UINT64_C(0xbf58476d1ce4e5b9); + x = (x ^ (x >> 27)) * UINT64_C(0x94d049bb133111eb); + x = x ^ (x >> 31); + return x; +} + // Convert a 64-bit hash value into a floating point number between 0 and 1. double hash_to_double(uint64_t hash); @@ -220,6 +301,14 @@ eng::string repeat_string(const eng::string &a, int n); eng::string tolower(eng::string input); eng::string toupper(eng::string input); +// Convert a codepoint number into a utf8 string. +// If the codepoint is invalid, returns empty string. +eng::string get_codepoint_utf8(int32_t cp); + +// Write a codepoint in utf8 to a stream. +// If the codepoint is invalid, writes nothing and returns false. +bool write_codepoint_utf8(int32_t cp, std::ostream *out); + // Calculate distance between two points double distance_squared(double x1, double y1, double x2, double y2); @@ -247,15 +336,12 @@ struct XYZ { XYZ(float ix, float iy, float iz) { x=ix; y=iy; z=iz; } bool operator ==(const XYZ &o) const { return x==o.x && y == o.y && z==o.z; } bool operator !=(const XYZ &o) const { return x!=o.x || y != o.y || z!=o.z; } + XYZ operator -(const XYZ &o) const { return XYZ(x-o.x, y-o.y, z-o.z); } + XYZ operator +(const XYZ &o) const { return XYZ(x+o.x, y+o.y, z+o.z); } + XYZ operator *(float scale) const { return XYZ(x*scale, y*scale, z*scale); } eng::string debug_string() const; }; -// These are formatting directives that can be sent to a std::ostream. -class hex64 {}; -class hex32 {}; -class hex16 {}; -class hex8 {}; - class NullStreamBuffer : public std::streambuf { public: @@ -265,24 +351,69 @@ public: // send_to_stream: send all arguments to the specified stream. inline void send_to_stream(std::ostream &os) {} template -inline void send_to_stream(std::ostream &os, ARG arg, REST & ... rest) { +inline void send_to_stream(std::ostream &os, const ARG &arg, const REST & ... rest) { os << arg; send_to_stream(os, rest...); } // ss: convert all arguments to a string by sending them to a stringstream. template -inline eng::string ss(ARGS & ... args) { +inline eng::string ss(const ARGS & ... args) { eng::ostringstream oss; send_to_stream(oss, args...); return oss.str(); } +// A better API than std::setfill, std::hex, std::setw, std::setprecision +// +// Usage examples: +// std::cout << util::hex.width(5).fill('0').val(123) +// std::cout << util::dec.fill('$').precision(val(123) +// +// The reason that other API is bad is that it can leave std::cout +// in an unpredictable state. This API always leaves the stream clean. +// +template +class FormattedNumber { +public: + VALUE value_; + bool hex_; + int width_; + char fill_; + int precision_; + + constexpr FormattedNumber(VALUE v, bool h, int w, char f, int p) + : value_(v), hex_(h), width_(w), fill_(f), precision_(p) {} + + constexpr FormattedNumber width(int w) const { return FormattedNumber(value_, hex_, w, fill_, precision_); } + constexpr FormattedNumber fill(char f) const { return FormattedNumber(value_, hex_, width_, f, precision_); } + constexpr FormattedNumber precision(int p) const { return FormattedNumber(value_, hex_, width_, fill_, p); } + + template + constexpr FormattedNumber val(NVALUE v) const { return FormattedNumber(v, hex_, width_, fill_, precision_); } +}; + +constexpr auto hex = FormattedNumber(0, true, 0, '0', 6); +constexpr auto hex8 = FormattedNumber(0, true, 2, '0', 6); +constexpr auto hex16 = FormattedNumber(0, true, 4, '0', 6); +constexpr auto hex32 = FormattedNumber(0, true, 8, '0', 6); +constexpr auto hex64 = FormattedNumber(0, true, 16, '0', 6); +constexpr auto dec = FormattedNumber(0, false, 0, ' ', 6); + } // namespace util -std::ostream &operator<<(std::ostream &oss, const util::hex64 &v); -std::ostream &operator<<(std::ostream &oss, const util::hex32 &v); -std::ostream &operator<<(std::ostream &oss, const util::hex16 &v); -std::ostream &operator<<(std::ostream &oss, const util::hex8 &v); +template +inline std::ostream &operator<<(std::ostream &oss, util::FormattedNumber n) { + if (n.hex_) oss << std::hex; + else oss << std::dec; + oss << std::setprecision(n.precision_) << std::setfill(n.fill_) << std::setw(n.width_) << n.value_; + oss << std::dec << std::setfill(' ') << std::setprecision(6); + return oss; +} + +inline std::ostream &operator<<(std::ostream &oss, const util::XYZ &xyz) { + oss << xyz.x << "," << xyz.y << "," << xyz.z; + return oss; +} #endif // UTIL_HPP diff --git a/luprex/core/cpp/world-accessor.cpp b/luprex/core/cpp/world-accessor.cpp index e3d7b7cf..c9772913 100644 --- a/luprex/core/cpp/world-accessor.cpp +++ b/luprex/core/cpp/world-accessor.cpp @@ -265,11 +265,17 @@ LuaDefine(tangible_near, "tan,radius,omit_nowhere,omit_self", LuaStack LS(L, ltan, lradius, lomit_nowhere, lomit_self, list); World *w = World::fetch_global_pointer(L); Tangible *tan = w->tangible_get(LS, ltan); - double radius = LS.cknumber(lradius); - bool omit_nowhere = LS.ckboolean(lomit_nowhere); - bool omit_self = LS.ckboolean(lomit_self); const AnimStep &aqback = tan->anim_queue_.back(); - util::IdVector idv = w->plane_map_.scan_radius(aqback.plane(), aqback.xyz().x, aqback.xyz().y, radius, omit_nowhere, tan->id(), omit_self); + + PlaneScan scan; + scan.set_plane(aqback.plane()); + scan.set_bbox_given_center_radius(aqback.xyz(), LS.cknumber(lradius)); + scan.set_shape(PlaneScan::SPHERE); + scan.set_sorted(true); + scan.set_near(tan->id(), !LS.ckboolean(lomit_self)); + scan.set_omit_nowhere(LS.ckboolean(lomit_nowhere)); + + util::IdVector idv = w->plane_map_.scan(scan); tangible_getall(LS, list, idv); return LS.result(); } @@ -282,16 +288,98 @@ LuaDefine(tangible_scan, "plane,x,y,radius,omit_nowhere", LuaRet list; LuaStack LS(L, lplane, lx, ly, lradius, lomit_nowhere, list); World *w = World::fetch_global_pointer(L); - eng::string plane = LS.ckstring(lplane); - double x = LS.cknumber(lx); - double y = LS.cknumber(ly); - double radius = LS.cknumber(lradius); - bool omit_nowhere = LS.ckboolean(lomit_nowhere); - util::IdVector idv = w->plane_map_.scan_radius(plane, x, y, radius, omit_nowhere, 0, false); + + PlaneScan scan; + scan.set_plane(LS.ckstring(lplane)); + scan.set_bbox_given_center_radius(util::XYZ(LS.cknumber(lx), LS.cknumber(ly), 0), LS.cknumber(lradius)); + scan.set_shape(PlaneScan::SPHERE); + scan.set_sorted(true); + scan.set_omit_nowhere(LS.ckboolean(lomit_nowhere)); + + util::IdVector idv = w->plane_map_.scan(scan); tangible_getall(LS, list, idv); return LS.result(); } +LuaDefine(tangible_find, "config", + "|Find tangibles by their location." + "|" + "|There are multiple ways to specify the plane, the bounding" + "|box, and the shape of the search. The most basic is to" + "|include these parameters in the table:" + "|" + "| plane : the plane to search (a string)" + "| centerx : x-coordinate of the center of the search" + "| centery : y-coordinate of the center of the search" + "| centerz : z-coordinate of the center of the search" + "| radius : the radius of the search" + "| shape : 'box', 'sphere', or 'cylinder'" + "|" + "|Shape has a default: 'sphere'. The other parameters do" + "|not have default values." + "|" + "|Instead of specifying the radius as a single float," + "|you may optionally specify separate radii for each dimension." + "|" + "| radiusx : the radius in the x-dimension." + "| radiusy : the radius in the y-dimension." + "| radiusz : the radius in the z-dimension." + "|" + "|If you specify different radii in each dimension, then the" + "|'sphere' shape will actually be a spheroid, and the 'cylinder'" + "|shape will actually be a cylindroid." + "|" + "|Instead of specifying the center and plane, you can specify" + "|a tangible to search near:" + "|" + "| near : a tangible in whose vicinity the search occurs" + "|" + "|If you specify 'near', then by default, the near tangible is" + "|filtered out of the search. If you want to include it in the" + "|results, set the 'include' flag to true. In this case, the" + "|near tangible will always be the first search result:" + "|" + "| include : include the 'near' object in the results" + "|" + "|If you want to search an entire plane, you can use the" + "|wholeplane option, which replaces all the other options:" + "|" + "| wholeplane: the name of the plane to scan in its entirety" + "|" + "|It is valid to use 0.0 as the radius. If you do so, then only" + "|objects at the exact center you specify will be found." + "|" + "|If you are making a 2D game, it works fine to set all object Z" + "|coordinates to zero, and then do searches with centerz=0.0" + "|and radiusz=0.0." + "|") { + LuaArg config; + LuaRet result; + LuaStack LS(L, config, result); + PlaneScan scan; + eng::string error = scan.configure(LS, config); + if (!error.empty()) { + luaL_error(L, "%s", error.c_str()); + } + // When the configure routine sees the 'near' flag, it stores the tangible + // ID, but not the center and plane, because doing so would require it to + // know about world models. We have to handle center and plane for 'near' + // separately. + World *w = World::fetch_global_pointer(L); + int64_t near = scan.near(); + if (near != 0) { + Tangible *t = w->tangible_get(near); + assert(t != nullptr); // Should never happen. + const AnimStep &aqback = t->anim_queue_.back(); + scan.set_plane(aqback.plane()); + scan.set_center(aqback.xyz()); + } + // Do the scan. + util::IdVector idv = w->plane_map_.scan(scan); + tangible_getall(LS, result, idv); + return LS.result(); +} + LuaDefine(tangible_start, "tangible,function,arg1,arg2...", "|Start a thread." "|" @@ -462,6 +550,9 @@ LuaDefine(math_random, "(args...)", "| (high) - an int between 1 and high inclusive" "| (low, high) - an int between low and high inclusive" "|" + "|You may also pass in a randomstate table" + "|as an optional first argument." + "|" "|math.random tries to cooperate with predictive" "|reexecution to be as predictable as possible." "|To achieve predictability, we used an ad-hoc" @@ -654,9 +745,7 @@ LuaDefine(doc, "function", return LS.result(); } -LuaDefine(http_get, "request", - "|Make an HTTP GET request. Returns an HTTP response." - "|See doc(http.clientrequest) and doc(http.clientresponse).") { +int lfn_http_request(lua_State *L, const char *method) { World *w = World::fetch_global_pointer(L); w->guard_blockable(L, "http.get"); @@ -667,7 +756,7 @@ LuaDefine(http_get, "request", // Parse the request and make sure it's valid. // If not, immediately pass a '400 bad request' back to lua. - req.set_method("GET"); + req.set_method(method); req.set_config(LS, request); req.set_defaults(); eng::string error = req.check(); @@ -686,4 +775,22 @@ LuaDefine(http_get, "request", // Block. return lua_yield(L, 0); -} \ No newline at end of file +} + +LuaDefine(http_get, "request", + "|Make an HTTP GET request. Returns an HTTP response." + "|See doc(http.clientrequest) and doc(http.clientresponse).") { + return lfn_http_request(L, "GET"); +} + +LuaDefine(http_head, "request", + "|Make an HTTP HEAD request. Returns an HTTP response." + "|See doc(http.clientrequest) and doc(http.clientresponse).") { + return lfn_http_request(L, "HEAD"); +} + +LuaDefine(http_post, "request", + "|Make an HTTP POST request. Returns an HTTP response." + "|See doc(http.clientrequest) and doc(http.clientresponse).") { + return lfn_http_request(L, "POST"); +} diff --git a/luprex/core/cpp/world-core.cpp b/luprex/core/cpp/world-core.cpp index 186a68a0..f889d7f5 100644 --- a/luprex/core/cpp/world-core.cpp +++ b/luprex/core/cpp/world-core.cpp @@ -220,26 +220,23 @@ void World::tangible_delete(int64_t id) { LS.result(); } -util::IdVector World::get_near_unsorted(int64_t player_id, float radius, bool exclude_nowhere, bool omit_player) const { +util::IdVector World::get_near(int64_t player_id, float radius, bool exclude_nowhere, bool omit_player, bool sorted) const { const Tangible *player = tangible_get(player_id); if (player == nullptr) { return IdVector(); } - // Find out where's the center of the world. + // Find out where the player is. const AnimStep &aqback = player->anim_queue_.back(); - return plane_map_.scan_radius_unsorted(aqback.plane(), aqback.xyz().x, aqback.xyz().y, radius, exclude_nowhere, player_id, omit_player); -} -util::IdVector World::get_near(int64_t player_id, float radius, bool exclude_nowhere, bool omit_player) const { - const Tangible *player = tangible_get(player_id); - if (player == nullptr) { - return IdVector(); - } - - // Find out where's the center of the world. - const AnimStep &aqback = player->anim_queue_.back(); - return plane_map_.scan_radius(aqback.plane(), aqback.xyz().x, aqback.xyz().y, radius, exclude_nowhere, player_id, omit_player); + PlaneScan scan; + scan.set_plane(aqback.plane()); + scan.set_bbox_given_center_radius(aqback.xyz(), radius); + scan.set_shape(PlaneScan::SPHERE); + scan.set_sorted(sorted); + scan.set_omit_nowhere(exclude_nowhere); + scan.set_near(player_id, !omit_player); + return plane_map_.scan(scan); } World::Redirects World::fetch_redirects() { @@ -301,6 +298,7 @@ eng::string World::probe_lua(int64_t actor_id, const eng::string &lua) { for (int i = top + 1; i <= lua_gettop(L); i++) { LuaSpecial root(i); pprint(LS, root, true, ostream); + // TODO: this endl is unnecessary if we just printed a newline. (*ostream) << std::endl; } } else { diff --git a/luprex/core/cpp/world-difftab.cpp b/luprex/core/cpp/world-difftab.cpp index 1dad1037..59f0c10f 100644 --- a/luprex/core/cpp/world-difftab.cpp +++ b/luprex/core/cpp/world-difftab.cpp @@ -56,6 +56,10 @@ static bool equivalent_values(LuaStack &MLS, LuaSlot mval, LuaSlot mtnmap, if (SLS.type(sval) != LUA_TSTRING) return false; return MLS.ckstring(mval) == SLS.ckstring(sval); } + case LUA_TLIGHTUSERDATA: { + if (SLS.type(sval) != LUA_TLIGHTUSERDATA) return false; + return MLS.cktoken(mval) == SLS.cktoken(sval); + } case LUA_TFUNCTION: { // Cannot really compare. Just return true if the types match. return SLS.type(sval) == MLS.type(mval); @@ -105,6 +109,11 @@ static void transmit_value(LuaStack &MLS, LuaSlot mval, LuaSlot mtnmap, StreamBu sb->write_string(MLS.ckstring(mval)); return; } + case LUA_TLIGHTUSERDATA: { + sb->write_uint8(LUA_TLIGHTUSERDATA); + sb->write_uint64(MLS.cktoken(mval).value); + return; + } case LUA_TT_GENERAL: { int midx = get_table_number(MLS, mval, mtnmap); if (midx == 0) { @@ -151,6 +160,10 @@ static void transmit_value_debug_string(StreamBuffer *sb, eng::ostringstream &os oss << sb->read_string(); return; } + case LUA_TLIGHTUSERDATA: { + LuaToken token(sb->read_uint64()); + oss << "[" << token.str() << "]"; + } case LUA_TT_GENERAL: { oss << "table " << sb->read_int32(); return; @@ -270,6 +283,12 @@ static void set_transmitted_value(LuaStack &LS, LuaSlot tangibles, LuaSlot ntmap LS.set(target, value); return; } + case LUA_TLIGHTUSERDATA: { + LuaToken value(sb->read_uint64()); + DebugLine(dbc) << dbinfo << "[" << value.str() << "]"; + LS.set(target, value); + return; + } case LUA_TT_GENERAL: { int index = sb->read_int32(); DebugLine(dbc) << dbinfo << "table " << index; diff --git a/luprex/core/cpp/world-diffxmit.cpp b/luprex/core/cpp/world-diffxmit.cpp index fdefe54a..abb60d52 100644 --- a/luprex/core/cpp/world-diffxmit.cpp +++ b/luprex/core/cpp/world-diffxmit.cpp @@ -3,8 +3,8 @@ util::IdVector World::get_visible_union(int64_t actor_id, World *master) { return util::sort_union_id_vectors( - master->get_near_unsorted(actor_id, RadiusVisibility, true, false), - get_near_unsorted(actor_id, RadiusVisibility, true, false)); + master->get_near(actor_id, RadiusVisibility, true, false, false), + get_near(actor_id, RadiusVisibility, true, false, false)); } int64_t World::patch_actor(StreamBuffer *sb, DebugCollector *dbc) { @@ -169,7 +169,7 @@ void World::patch_luatabs(StreamBuffer *sb, DebugCollector *dbc) { int64_t actor_id = sb->read_int64(); util::HashValue closehash = sb->read_hashvalue(); int ncreate = sb->read_int32(); - util::IdVector closetans = get_near(actor_id, RadiusClose, true, false); + util::IdVector closetans = get_near(actor_id, RadiusClose, true, false, true); assert(closehash == util::hash_id_vector(closetans)); number_lua_tables(closetans); create_new_tables(ncreate); @@ -183,8 +183,8 @@ void World::diff_luatabs(int64_t actor_id, World *master, StreamBuffer *xsb) { StreamBuffer tsb; // Calculate the set of close tangibles. - util::IdVector closetans = master->get_near(actor_id, RadiusClose, true, false); - assert(get_near(actor_id, RadiusClose, true, false) == closetans); + util::IdVector closetans = master->get_near(actor_id, RadiusClose, true, false, true); + assert(get_near(actor_id, RadiusClose, true, false, true) == closetans); util::HashValue closehash = util::hash_id_vector(closetans); // Number and pair tables in the synchronous and master model. @@ -250,7 +250,7 @@ void World::diff_tanclass(int64_t actor_id, World *master, StreamBuffer *xsb) { // Calculate the set of close tangibles. // TODO: we've already calculated this in an earlier function. This is wasteful. - util::IdVector closetans = master->get_near(actor_id, RadiusClose, true, false); + util::IdVector closetans = master->get_near(actor_id, RadiusClose, true, false, true); tsb.write_int32(0); int write_count_after = tsb.total_writes(); diff --git a/luprex/core/cpp/world-testing.cpp b/luprex/core/cpp/world-testing.cpp index 7e99c2c3..0d777d08 100644 --- a/luprex/core/cpp/world-testing.cpp +++ b/luprex/core/cpp/world-testing.cpp @@ -39,7 +39,7 @@ eng::string World::tangible_ids_debug_string() const { eng::string World::tangibles_near_debug_string(int64_t actor, int64_t distance) { eng::ostringstream result; - for (int64_t id : get_near(actor, distance, true, false)) { + for (int64_t id : get_near(actor, distance, true, false, true)) { const Tangible *tan = tangible_get(id); const AnimStep &aqback = tan->anim_queue_.back(); result << id << ": " << aqback.graphic() << " " << aqback.plane() << " " << aqback.xyz().debug_string() << std::endl; diff --git a/luprex/core/cpp/world.hpp b/luprex/core/cpp/world.hpp index 61c2f82b..b506d91d 100644 --- a/luprex/core/cpp/world.hpp +++ b/luprex/core/cpp/world.hpp @@ -122,10 +122,10 @@ public: // // Get a list of the tangibles that are near the player. If 'exclude_nowhere' is // true, exclude any tangibles on the nowhere plane (but still include the player himself). - // The unsorted version returns the tangibles in an unpredictable order. + // The unsorted version returns the tangibles in an unpredictable order. If sorted + // is false, return them in an unpredictable order. // - IdVector get_near(int64_t player_id, float radius, bool exclude_nowhere, bool omit_player) const; - IdVector get_near_unsorted(int64_t player_id, float radius, bool exclude_nowhere, bool omit_player) const; + IdVector get_near(int64_t player_id, float radius, bool exclude_nowhere, bool omit_player, bool sorted) const; // Make a tangible. // @@ -552,12 +552,13 @@ private: friend int lfn_tangible_nopredict(lua_State *L); friend int lfn_tangible_near(lua_State *L); friend int lfn_tangible_scan(lua_State *L); + friend int lfn_tangible_find(lua_State *L); friend int lfn_tangible_start(lua_State *L); friend int lfn_math_random(lua_State *L); friend int lfn_math_randomstate(lua_State *L); friend int lfn_wait(lua_State *L); friend int lfn_nopredict(lua_State *L); - friend int lfn_http_get(lua_State *L); + friend int lfn_http_request(lua_State *L, const char *method); }; using UniqueWorld = std::unique_ptr; diff --git a/luprex/core/wrap/wrap-bytell-hash-map.hpp b/luprex/core/wrap/wrap-bytell-hash-map.hpp new file mode 100644 index 00000000..23bf3ed4 --- /dev/null +++ b/luprex/core/wrap/wrap-bytell-hash-map.hpp @@ -0,0 +1,14 @@ +#ifndef WRAP_BYTELL_HASH_MAP_HPP +#define WRAP_BYTELL_HASH_MAP_HPP + +#include "eng-malloc.hpp" +#include "bytell-hash-map.hpp" + +namespace eng { +template, class E=std::equal_to> +class bytell_hash_map : public ska::bytell_hash_map>>, public eng::opnew { + using ska::bytell_hash_map>>::bytell_hash_map; +}; +} // namespace eng + +#endif // WRAP_BYTELL_HASH_MAP_HPP