Overhaul animation queues so the queue is stored as a single encoded string.

luprex/cpp/core/animqueue.cpp

@@ -9,6 +9,9 @@
 #include <cmath>
 #include <cstdlib>
 
+util::SharedStdString make_shared_string(const StreamBuffer &sb) {
+    return std::make_shared<std::string>(sb.view());
+}
 
 static const char *vtname(AnimValueType vt) {
     switch (vt) {
@@ -22,7 +25,7 @@ static const char *vtname(AnimValueType vt) {
 }
 
 
-uint64_t hash_encoding(uint64_t prev, std::string_view s) {
+uint64_t hash_encstep(uint64_t prev, std::string_view s) {
     return util::hash_string(util::HashValue(123, prev), s).first;
 }
 
@@ -130,6 +133,31 @@ static void parse_value(std::string_view vstr, AnimValue *v) {
     v->set_string(vstr);
 }
 
+
+static AnimValue parse_anim_value(LuaCoreStack &LS, LuaSlot val, LuaSlot tmp) {
+    AnimValue result;
+    if (LS.isboolean(val)) {
+        result.set_boolean(LS.ckboolean(val));
+    } else if (LS.isnumber(val)) {
+        result.set_number(LS.cknumber(val));
+    } else if (LS.isstring(val)) {
+        result.set_string(LS.ckstring(val));
+    } else if (LS.istable(val)) {
+        util::DXYZ xyz;
+        LS.rawget(tmp, val, 1);
+        if (!LS.isnumber(tmp)) return result;
+        xyz.x = LS.cknumber(tmp);
+        LS.rawget(tmp, val, 2);
+        if (!LS.isnumber(tmp)) return result;
+        xyz.y = LS.cknumber(tmp);
+        LS.rawget(tmp, val, 3);
+        if (!LS.isnumber(tmp)) return result;
+        xyz.z = LS.cknumber(tmp);
+        result.set_xyz(xyz);
+    }
+    return result;
+}
+
 void AnimState::set_persistent(const eng::string &name) {
     map_[name].persistent = true;
 }
@@ -303,31 +331,6 @@ eng::string AnimState::add_defaults(const AnimState *other) {
 }
 
 
-static AnimValue parse_anim_value(LuaCoreStack &LS, LuaSlot val, LuaSlot tmp) {
-    AnimValue result;
-    if (LS.isboolean(val)) {
-        result.set_boolean(LS.ckboolean(val));
-    } else if (LS.isnumber(val)) {
-        result.set_number(LS.cknumber(val));
-    } else if (LS.isstring(val)) {
-        result.set_string(LS.ckstring(val));
-    } else if (LS.istable(val)) {
-        util::DXYZ xyz;
-        LS.rawget(tmp, val, 1);
-        if (!LS.isnumber(tmp)) return result;
-        xyz.x = LS.cknumber(tmp);
-        LS.rawget(tmp, val, 2);
-        if (!LS.isnumber(tmp)) return result;
-        xyz.y = LS.cknumber(tmp);
-        LS.rawget(tmp, val, 3);
-        if (!LS.isnumber(tmp)) return result;
-        xyz.z = LS.cknumber(tmp);
-        result.set_xyz(xyz);
-    }
-    return result;
-}
-
-
 eng::string AnimState::apply_lua(LuaCoreStack &LS0, LuaSlot tab, bool setpersist) {
     LuaVar key, val, tmp;
     LuaExtStack LS(LS0.state(), key, val, tmp);
@@ -420,6 +423,54 @@ void AnimCoreState::decode(std::string_view s) {
     }
 }
 
+
+struct StepBreakout {
+    uint64_t hash;
+    std::string_view encstep;
+    StepBreakout(uint64_t h, std::string_view e) : hash(h), encstep(e) {}
+};
+
+using StepBreakoutVec = eng::vector<StepBreakout>;
+
+StepBreakoutVec encqueue_breakout(std::string_view encqueue) {
+    StepBreakoutVec result;
+    StreamBuffer sb(encqueue);
+    while (!sb.empty()) {
+        uint64_t hash = sb.read_uint64();
+        std::string_view encstep = sb.read_string_view();
+        result.emplace_back(hash, encstep);
+    }
+    return result;
+}
+
+// Just return the hash of the very last step.  (Steps are stored last to first).
+static uint64_t encqueue_final_hash(std::string_view encqueue) {
+    StreamBuffer sb(encqueue);
+    uint64_t hash = sb.read_uint64();
+    return hash;
+}
+
+static std::string_view encqueue_final_encstep(std::string_view encqueue) {
+    StreamBuffer sb(encqueue);
+    sb.read_uint64();
+    std::string_view result = sb.read_string_view();
+    return result;
+}
+
+// Return the encqueue for the first N steps.  If there aren't that
+// many steps, then just return them all.
+std::string_view encqueue_firstn(std::string_view encqueue, int n) {
+    StreamBuffer sb(encqueue);
+    while ((n > 0) && (!sb.empty())) {
+        sb.read_uint64();
+        uint64_t slen = sb.read_length();
+        sb.read_bytes(slen);
+        n -= 1;
+    }
+    return encqueue.substr(0, sb.total_reads());
+}
+
+
 AnimQueue::AnimQueue() {
     size_limit_ = 10; // Default size limit.
     clear();
@@ -431,53 +482,39 @@ void AnimQueue::clear() {
 }
 
 void AnimQueue::clear(const AnimState &state) {
-    steps_.clear();
+    StreamBuffer result;
-    eng::string encoding = state.encode();
+    eng::string encstep = state.encode();
-    uint64_t hash = hash_encoding(0, encoding);
+    uint64_t hash = hash_encstep(0, encstep);
-    steps_.emplace_back(encoding, hash);
+    result.write_uint64(hash);
+    result.write_string(encstep);
+    encqueue_ = make_shared_string(result);
 }
 
-void AnimQueue::set_limit(int n) {
+void AnimQueue::set_limit(int nkeep) {
-    assert((n >= 2) && (n <= 250));
+    assert((nkeep >= 2) && (nkeep <= 250));
-    size_limit_ = n;
+    size_limit_ = nkeep;
-    if (int(steps_.size()) > n) {
+    encqueue_ = std::make_shared<std::string>(encqueue_firstn(*encqueue_, nkeep));
-        while (int(steps_.size()) > n) {
-            steps_.pop_front();
-        }
-    }
 }
 
 void AnimQueue::add(const AnimState &state) {
-    eng::string encoding = state.encode();
+    uint64_t previoushash = encqueue_final_hash(*encqueue_);
-    uint64_t hash = hash_encoding(steps_.back().hash, encoding);
+    eng::string encstep = state.encode();
-    steps_.emplace_back(encoding, hash);
+    uint64_t hash = hash_encstep(previoushash, encstep);
-    set_limit(size_limit_);
+    StreamBuffer result;
+    result.write_uint64(hash);
+    result.write_string(encstep);
+    result.write_bytes(encqueue_firstn(*encqueue_, size_limit_ - 1));
+    encqueue_ = make_shared_string(result);
 }
 
 void AnimQueue::serialize(StreamBuffer *sb) const {
     sb->write_uint8(size_limit_);
-    sb->write_uint8(steps_.size());
+    sb->write_string(*encqueue_);
-    sb->write_uint64(steps_[0].hash);
-    for (const AnimQueue::Step &step : steps_) {
-        sb->write_string(step.encoding);
-    }
 }
 
 void AnimQueue::deserialize(StreamBuffer *sb) {
     size_limit_ = sb->read_uint8();
-    int nsteps = sb->read_uint8();
+    encqueue_ = std::make_shared<std::string>(sb->read_string_view());
-    uint64_t firsthash = sb->read_uint64();
-    assert ((nsteps >= 2) && (nsteps <= size_limit_) && (size_limit_ >= 2) && (size_limit_ <= 250));
-    steps_.resize(nsteps);
-    for (int i = 0; i < nsteps; i++) {
-        AnimQueue::Step &step = steps_[i];
-        step.encoding = sb->read_string();
-        if (i == 0) {
-            step.hash = firsthash;
-        } else {
-            step.hash = hash_encoding(steps_[i-1].hash, step.encoding);
-        }
-    }
 }
 
 bool AnimQueue::diff(const AnimQueue &auth, StreamBuffer *sb) const {
@@ -488,28 +525,10 @@ bool AnimQueue::diff(const AnimQueue &auth, StreamBuffer *sb) const {
         return false;
     }
 
-    assert(auth.steps_.size() < 255);
+    // TODO: maybe send less data?
-    sb->write_uint8(auth.steps_.size());
+    sb->write_uint8(0);
-    sb->write_uint8(auth.size_limit_);
+    sb->write_uint32(auth.size_limit_);
-    sb->write_uint64(auth.steps_.front().hash);
+    sb->write_string(*auth.encqueue_);
-
-    // Index all known elements by text.
-    eng::map<eng::string, int> index;
-    for (int i = 0; i < int(steps_.size()); i++) {
-        index[steps_[i].encoding] = i;
-    }
-
-    // Write the encoded elements.
-    for (int i = 0; i < int(auth.steps_.size()); i++) {
-        const AnimQueue::Step &step = auth.steps_[i];
-        auto iter = index.find(step.encoding);
-        if (iter == index.end()) {
-            sb->write_uint8(255);
-            sb->write_string(step.encoding);
-        } else {
-            sb->write_uint8(iter->second);
-        }
-    }
     return true;
 }
 
@@ -520,68 +539,46 @@ void AnimQueue::patch(StreamBuffer *sb, DebugCollector *dbc) {
     }
     DebugLine(dbc) << "AnimQueue modified";
 
-    size_limit_ = sb->read_uint8();
+    size_limit_ = sb->read_uint32();
-    uint64_t firsthash = sb->read_uint64();
+    std::string_view steps = sb->read_string_view();
-
+    encqueue_ = std::make_shared<std::string>(steps);
-    assert ((nsteps >= 2) && (nsteps <= size_limit_) && (size_limit_ >= 2) && (size_limit_ <= 250));
-
-    // Decode the diff, stop at eof.
-    eng::deque<AnimQueue::Step> old = std::move(steps_);
-    steps_.resize(nsteps);
-    for (int i = 0; i < nsteps; i++) {
-        AnimQueue::Step &step = steps_[i];
-        uint8_t index = sb->read_uint8();
-        if (index < 255) {
-            assert(index < old.size());
-            step.encoding = old[index].encoding;
-        } else {
-            step.encoding = sb->read_string();
-        }
-        if (i == 0) {
-            step.hash = firsthash;
-        } else {
-            step.hash = hash_encoding(steps_[i-1].hash, step.encoding);
-        }
-    }
 }
 
 bool AnimQueue::exactly_equal(const AnimQueue &other) const {
     if (size_limit_ != other.size_limit_) return false;
-    if (steps_.size() != other.steps_.size()) return false;
+    if (*encqueue_ != *other.encqueue_) return false;
-    for (int i = 0; i < int(steps_.size()); i++) {
-        const AnimQueue::Step &step = steps_[i];
-        const AnimQueue::Step &otherstep = other.steps_[i];
-        if (step.hash != otherstep.hash) return false;
-        if (step.encoding != otherstep.encoding) return false;
-    }
     return true;
 }
 
 bool AnimQueue::exactly_equal_fast(const AnimQueue &other) const {
     if (size_limit_ != other.size_limit_) return false;
-    if (steps_.size() != other.steps_.size()) return false;
+    if (encqueue_->size() != other.encqueue_->size()) return false;
-    if (steps_.back().hash != other.steps_.back().hash) return false;
+    if (encqueue_->compare(0, 8, *other.encqueue_) != 0) return false;
     return true;
 }
 
 eng::string AnimQueue::steps_debug_string() const {
+    StepBreakoutVec breakout = encqueue_breakout(*encqueue_);
     eng::ostringstream oss;
     bool first = true;
-    for (const AnimQueue::Step &step : steps_) {
+    for (int i = breakout.size() - 1; i >= 0; i--) {
+        const StepBreakout &step = breakout[i];
         if (!first) oss << "; ";
         first = false;
-        AnimState state(step.encoding);
+        AnimState state(step.encstep);
         state.print_debug_string(oss);
     }
     return oss.str();
 }
 
 eng::string AnimQueue::full_debug_string() const {
+    StepBreakoutVec breakout = encqueue_breakout(*encqueue_);
     eng::ostringstream oss;
     oss << "limit=" << size_limit();
-    for (const AnimQueue::Step &step : steps_) {
+    for (int i = breakout.size() - 1; i >= 0; i--) {
+        const StepBreakout &step = breakout[i];
         oss << "; ";
-        AnimState state(step.encoding);
+        AnimState state(step.encstep);
         state.print_debug_string(oss);
     }
     return oss.str();
@@ -590,38 +587,30 @@ eng::string AnimQueue::full_debug_string() const {
 // Get the final entry, xyz and plane only.
 //
 AnimCoreState AnimQueue::get_final_core_state() const {
+    std::string_view encstep = encqueue_final_encstep(*encqueue_);
     AnimCoreState result;
-    result.decode(steps_.back().encoding);
+    result.decode(encstep);
     return result;
 }
 
 // Get the final entry, all persistent variables.
 //
 AnimState AnimQueue::get_final_persistent() const {
+    std::string_view encstep = encqueue_final_encstep(*encqueue_);
     AnimState result;
-    result.decode_persistent(steps_.back().encoding);
+    result.decode_persistent(encstep);
     return result;
 }
 
 // Get the final entry, everything persistent and non-persistent.
 //
 AnimState AnimQueue::get_final_everything() const {
+    std::string_view encstep = encqueue_final_encstep(*encqueue_);
     AnimState result;
-    result.decode(steps_.back().encoding);
+    result.decode(encstep);
     return result;
 }
 
-void AnimQueue::get_for_engine_wrapper(std::vector<EngineWrapper::AnimEntry> *into) const {
-    into->resize(steps_.size());
-    for (int i = 0; i < int(steps_.size()); i++) {
-        const AnimQueue::Step &step = steps_[i];
-        EngineWrapper::AnimEntry &entry = (*into)[i];
-        entry.hash = step.hash;
-        entry.data = step.encoding.c_str();
-        entry.size = step.encoding.size();
-    }
-}
-
 
 LuaDefine(unittests_animqueue, "", "some unit tests") {
     // Useful objects.
@@ -757,7 +746,7 @@ LuaDefine(unittests_animqueue, "", "some unit tests") {
     aqs.clear();
     sb.clear();
     aqs.diff(aq, &sb);
-    int difflen1 = sb.fill();
+    //int difflen1 = sb.fill();
     LuaAssert(L, !aqs.exactly_equal(aq));
     aqs.patch(&sb, nullptr);
     LuaAssert(L, aqs.exactly_equal(aq));
@@ -775,12 +764,14 @@ LuaDefine(unittests_animqueue, "", "some unit tests") {
     LuaAssertStrEq(L, aqs.full_debug_string(), "limit=10; action:jump xyz=4,5,6; plane=earth xyz=1,2,3; airline:southwest plane=moon");
     sb.clear();
     aqs.diff(aq, &sb);
-    int difflen2 = sb.fill();
+    //int difflen2 = sb.fill();
     LuaAssert(L, !aqs.exactly_equal(aq));
     aqs.patch(&sb, nullptr);
     LuaAssert(L, aqs.exactly_equal(aq));
     LuaAssertStrEq(L, aqs.full_debug_string(), "limit=10; plane=earth xyz=1,2,3; action:jump xyz=4,5,6; airline:southwest plane=moon; color=blue");
-    LuaAssert(L, difflen2 < (difflen1 / 2));
+
+    // TODO: if we make the diff routine more efficient, this should be true.
+    // LuaAssert(L, difflen2 < (difflen1 / 2));
 
     return 0;
 }

luprex/cpp/core/animqueue.hpp

@@ -18,14 +18,63 @@
 // Persistent values are retained from one animation step to the next,
 // nonpersistent values exist for one animation step only.
 //
-// Animation steps are stored encoded as strings, which is convenient for
-// passing the data to unreal, but it means that the animation step has to be
-// decoded whenever you want access to the key-value pairs.
-//
 // Each animation step has a hash value.  The hash value is generated
 // by mixing the hash value of the previous step with the hash value
 // of the encoded string of key-value pairs.
 //
+//
+///////////////////////////////////////////////////////////////////
+//
+// SERIALIZED STORAGE
+//
+// The entired animation queue is stored in a serialized format,
+// as a shared string.  This means that the animation queue can be
+// passed to the graphics engine as a single string.  This vastly
+// simplifies the API for passing the animation queue to the
+// graphics engine.  It also vastly reduces the amount of computation
+// done during the graphics engine probe: all we have to do is get
+// the already existing string and pass a reference to it.
+//
+// However, that means that when manipulating the animation queue,
+// we have to decode it, manipulate it, and then reencode it.  This
+// is a good tradeoff: we update animation queues rarely, compared
+// to how often we pass them to the graphics engine.
+//
+// So first, you need to know how to serialize a single animation
+// step.  Remember, an animation step consists of a list of key-value
+// pairs (see above).  The key-value pairs are serialized as follows:
+//
+//   for all key-value pairs do:
+//      write_string(key)
+//      write_bool(persistent)
+//      write_uint8(type) // T_STRING, T_NUMBER, T_BOOL, or T_XYZ
+//      switch type:
+//        T_STRING: write_string(value)
+//        T_NUMBER: write_double(value)
+//        T_BOOL: write_bool(value)
+//        T_XYZ: write_xyz(value)
+//
+// The encoded string produced by the loop above is called an "encstep".
+// That's short for "encoded animation step".  The encstep has a hash
+// value, which is a function that accepts the encstep and also the hash
+// of the previous encstep.  Note that the hash is not part of the encstep.
+//
+// An animation queue consists of a list of steps. Each step has a hash
+// and an encstep.  An animation queue is serialized as follows:
+//
+//   for all animation steps, starting with the most recent, do:
+//      write_uint64(hash)
+//      write_string(encstep)
+//
+// The encoded string produced by the loop above is called an "encqueue",
+// because it encodes everything in the animation queue (except for the
+// size limit, which is separate).
+//
+// Since the steps in an encqueue are stored most-recent first, if you
+// want some information about the most recent animation entry, you
+// don't need to decode the entire encqueue.  You only need to
+// decode the first step.
+//
 ///////////////////////////////////////////////////////////////////
 
 #ifndef ANIMQUEUE_HPP
@@ -40,7 +89,6 @@
 #include "streambuffer.hpp"
 #include "debugcollector.hpp"
 #include "util.hpp"
-#include "enginewrapper.hpp"
 
 #include <cassert>
 #include <ostream>
@@ -190,14 +238,6 @@ struct AnimCoreState
 };
 
 class AnimQueue : public eng::nevernew {
-private:
-    struct Step {
-        Step() { hash=0; }
-        Step(const eng::string &e, uint64_t h) : encoding(e), hash(h) {}
-        eng::string encoding;
-        uint64_t hash;
-    };
-
 public:
     // Construct an empty animation queue.
     // clears the state to a valid state.
@@ -250,10 +290,6 @@ public:
     eng::string steps_debug_string() const;
     eng::string full_debug_string() const;
 
-    // Get the final hash value.
-    //
-    uint64_t get_final_hash() const { return steps_.back().hash; }
-    
     // Get the final entry, xyz and plane only.
     //
     AnimCoreState get_final_core_state() const;
@@ -266,17 +302,22 @@ public:
     //
     AnimState get_final_everything() const;
     
-    // Get the contents of the animation queue for export to the engine wrapper.
+    // Get a serialized representation of the animation queue.
     //
-    // Caution: this exports pointers into existing allocated strings.
+    // Get the entire animation queue in a serialized format (encqueue).
-    // The pointers in this array are only valid until you modify the
+    // The string returned is a shared string.  No string copy
-    // animation queue.
+    // is made during this process.
     //
-    void get_for_engine_wrapper(std::vector<EngineWrapper::AnimEntry> *into) const;
+    util::SharedStdString get_encoded_queue() const { return encqueue_; }
 
 private:
     int size_limit_;
-    eng::deque<Step> steps_;
+
+    // Note: this is stored as a std::string, not an eng::string, because the
+    // ownership ends up being shared between us and the graphics engine. We
+    // can't have the graphics engine affecting the behavior of the engine heap.
+    //
+    util::SharedStdString encqueue_;
 };
 
 #endif // ANIMQUEUE_HPP

luprex/cpp/core/drivenengine.cpp

@@ -377,30 +377,21 @@ void DrivenEngine::drv_get_tangibles_near(uint64_t tanid, double rx, double ry,
     assert(hash1 == hash2);
 }
 
-void DrivenEngine::drv_get_animation_queue_hashes(uint32_t count, const int64_t *ids, uint64_t *hashes) {
+void DrivenEngine::drv_get_animation_queues(uint32_t count, const int64_t *ids, uint32_t *lengths, const char **strings) {
-    if (visible_world_ == 0) {
+    anim_queues_.resize(count);
+
+    if (visible_world_ == nullptr) {
+        util::SharedStdString empty = std::make_shared<std::string>("");
         for (int i = 0; i < int(count); i++) {
-            hashes[i] = 0;
+            anim_queues_[i] = empty;
         }
     } else {
-        visible_world_->get_animation_queue_hashes(count, ids, hashes);
+        visible_world_->get_encoded_animation_queues(count, ids, anim_queues_);
     }
-}
+    for (int i = 0; i < int(count); i++) {
-
+        lengths[i] = anim_queues_[i]->size();
-void DrivenEngine::drv_get_animation_queue(uint64_t tanid, uint32_t *count, AnimEntry **entries) {
+        strings[i] = anim_queues_[i]->c_str();
-    uint32_t hash1 = eng::memhash();
-    anim_queue_result_.clear();
-    if ((visible_world_ != 0) && (tanid != 0)) {
-        visible_world_->get_animation_queue(tanid, &anim_queue_result_);
     }
-    *count = anim_queue_result_.size();
-    if (count > 0) {
-        *entries = &(anim_queue_result_[0]);
-    } else {
-        *entries = nullptr;
-    }
-    uint32_t hash2 = eng::memhash();
-    assert(hash1 == hash2);
 }
 
 //////////////////////////////////////////////////////////////////////////////
@@ -532,13 +523,10 @@ static void drv_get_tangibles_near(EngineWrapper *w, uint64_t tanid, double rx,
     return w->engine->drv_get_tangibles_near(tanid, rx, ry, rz, count, ids);
 }
 
-static void drv_get_animation_queue_hashes(EngineWrapper *w, uint32_t count, const int64_t *ids, uint64_t *hashes) {
+static void drv_get_animation_queues(EngineWrapper *w, uint32_t count, const int64_t *ids, uint32_t *lengths, const char **strings) {
-    return w->engine->drv_get_animation_queue_hashes(count, ids, hashes);
+    return w->engine->drv_get_animation_queues(count, ids, lengths, strings);
 }
 
-static void drv_get_animation_queue(EngineWrapper *w, int64_t tanid, uint32_t *count, EngineWrapper::AnimEntry **entries) {
-    return w->engine->drv_get_animation_queue(tanid, count, entries);
-}
 
 //////////////////////////////////////////////////////////////////////////////
 //////////////////////////////////////////////////////////////////////////////
@@ -935,9 +923,8 @@ static void init_engine_wrapper_helper(EngineWrapper *w) {
     w->get_stop_driver = drv_get_stop_driver;
     w->get_actor_id = drv_get_actor_id;
     w->get_tangibles_near = drv_get_tangibles_near;
-    w->get_animation_queue_hashes = drv_get_animation_queue_hashes;
+    w->get_animation_queues = drv_get_animation_queues;
-    w->get_animation_queue = drv_get_animation_queue;
+    
-
     w->play_initialize = play_initialize;
     w->play_clear_new_outgoing = play_clear_new_outgoing;
     w->play_sent_outgoing = play_sent_outgoing;

luprex/cpp/core/drivenengine.hpp

@@ -125,8 +125,6 @@ using SharedChannel = std::shared_ptr<Channel>;
 
 class DrivenEngine : public eng::opnew {
 public:
-    using AnimEntry = EngineWrapper::AnimEntry;
-
     //////////////////////////////////////////////////////////////
     //
     // Build the named engine
@@ -284,8 +282,7 @@ public:
     bool drv_get_stop_driver() const;
     uint64_t drv_get_actor_id() const;
     void drv_get_tangibles_near(uint64_t tanid, double rx, double ry, double rz, uint32_t *count, int64_t **ids);
-    void drv_get_animation_queue_hashes(uint32_t count, const int64_t *ids, uint64_t *hashes);
+    void drv_get_animation_queues(uint32_t count, const int64_t *ids, uint32_t *lengths, const char **strings);
-    void drv_get_animation_queue(uint64_t tanid, uint32_t *count, AnimEntry **entries);
 
     void drv_initialize(uint32_t srcpklen, const char *srcpk, int argc, char **argv);
     void drv_clear_new_outgoing();
@@ -315,7 +312,7 @@ private:
     World *visible_world_;
     int64_t visible_actor_id_;
     util::IdVector scan_result_;
-    std::vector<AnimEntry> anim_queue_result_;
+    std::vector<util::SharedStdString> anim_queues_;
     bool rescan_lua_source_;
     double clock_;
     bool stop_driver_;

luprex/cpp/core/enginewrapper.hpp

@@ -31,22 +31,6 @@ struct EngineWrapper {
     PlayLogfile *wlog;
     ReplayLogfile *rlog;
 
-    //////////////////////////////////////////////////////////////////////////////
-    //////////////////////////////////////////////////////////////////////////////
-    //
-    // STRUCTURES
-    //
-    // These structures are used to return complicated values.
-    //
-    //////////////////////////////////////////////////////////////////////////////
-    //////////////////////////////////////////////////////////////////////////////
-
-    struct AnimEntry {
-        uint64_t    hash;
-        uint32_t    size;
-        const char *data;
-    };
-
     //////////////////////////////////////////////////////////////////////////////
     //////////////////////////////////////////////////////////////////////////////
     //
@@ -133,25 +117,19 @@ struct EngineWrapper {
     // Do a scan to find tangibles near the specified player.
     //
     // Returns a count and a pointer to an array of tangible IDs.  The returned
-    // pointer is valid until the next call into the engine.
+    // pointer is valid until the next call to get_tangibles_near.
     //
     void (*get_tangibles_near)(EngineWrapper *w, uint64_t tanid, double rx, double ry, double rz, uint32_t *count, int64_t **ids);
 
-    // Get the hash value of the final animation step for each tangible.
+    // Get the animation queues for the specified tangibles.
     //
-    // You must supply an array of tangible IDs.  The animation queue hash for
+    // You must supply an array of tangible IDs.  This returns the serialized
-    // each such tangible will be looked up.  You must supply a buffer to
+    // animation queues for all specified tangibles as strings.  The serialized
-    // hold the returned hashes.
+    // format is documented in header file animqueue.hpp.  The returned pointers
+    // remain valid until the next call to get_animation_queues.
     //
-    void (*get_animation_queue_hashes)(EngineWrapper *w, uint32_t count, const int64_t *ids, uint64_t *hashes);
+    void (*get_animation_queues)(EngineWrapper *w, uint32_t count, const int64_t *ids, uint32_t *lengths, const char **strings);
     
-    // Get the entire contents of an animation queue.
-    //
-    // Returns a count and a pointer to an array of AnimEntry.  The returned
-    // pointer is valid until the next call into the engine.
-    //
-    void (*get_animation_queue)(EngineWrapper *w, int64_t tanid, uint32_t *count, AnimEntry **buffer);
-
     //////////////////////////////////////////////////////////////////////////////
     //////////////////////////////////////////////////////////////////////////////
     //

luprex/cpp/core/streambuffer.cpp

@@ -169,6 +169,15 @@ void StreamBuffer::read_bytes_into(char *data, int64_t size) {
     read_cursor_ += size;
 }
 
+std::string_view StreamBuffer::read_string_view_limit(uint64_t limit) {
+    uint64_t length = read_length();
+    if (length > limit) throw StreamCorruption();
+    check_available(length);
+    std::string_view result(read_cursor_, length);
+    read_cursor_ += length;
+    return result;
+}
+
 void StreamBuffer::write_xyz(const util::XYZ &xyz) {
     make_space(12);
     memcpy(write_cursor_, &xyz.x, 4);

luprex/cpp/core/streambuffer.hpp

@@ -304,6 +304,11 @@ public:
     //
     void read_bytes_into(char *target, int64_t len);
     
+    // Read a string as a string_view.
+    //
+    std::string_view read_string_view_limit(uint64_t limit);
+    std::string_view read_string_view() { return read_string_view_limit(0x1000000); }
+
     // Read and write larger types.
     //
     // Throws StreamEof if the specified number of bytes aren't present.
@@ -385,7 +390,7 @@ public:
     // This is always false, because this module throws exceptions
     // when reading beyond EOF.
     bool read_beyond_eof() { return false; }
-    
+
 private:
     // Start and end of the allocated block.
     char *buf_lo_;

luprex/cpp/core/util.hpp

@@ -223,6 +223,8 @@ using StringSet = eng::set<eng::string>;
 using LuaSourceVec = eng::vector<StringPair>;
 using LuaSourcePtr = std::unique_ptr<LuaSourceVec>;
 using HashValue = std::pair<uint64_t, uint64_t>;
+using SharedStdString = std::shared_ptr<std::string>;
+using SharedStdStringVec = std::vector<SharedStdString>;
 
 // Ascii uppercase and lowercase.
 eng::string ascii_tolower(std::string_view c);

luprex/cpp/core/world-core.cpp

@@ -227,24 +227,19 @@ void World::get_near(int64_t player_id, float radius, bool exclude_nowhere, bool
     get_near(scan, into);
 }
 
-void World::get_animation_queue_hashes(uint32_t count, const int64_t *ids, uint64_t *hashes) {
+void World::get_encoded_animation_queues(uint32_t count, const int64_t *ids, util::SharedStdStringVec &into) {
+    util::SharedStdString empty = std::make_shared<std::string>("");
+    into.resize(count);
     for (int i = 0; i < int(count); i++) {
         Tangible *tan = tangible_get(ids[i]);
-        if (tan == 0) {
+        if (tan == nullptr) {
-            hashes[i] = 0;
+            into[i] = empty;
         } else {
-            hashes[i] = tan->anim_queue_.get_final_hash();
+            into[i] = tan->anim_queue_.get_encoded_queue();
         }
     }
 }
 
-void World::get_animation_queue(int64_t tanid, std::vector<EngineWrapper::AnimEntry> *into) {
-    Tangible *tan = tangible_get(tanid);
-    if (tan != nullptr) {
-        tan->anim_queue_.get_for_engine_wrapper(into);
-    }
-}
-
 World::Redirects World::fetch_redirects() {
     World::Redirects result = std::move(redirects_);
     redirects_.clear();

luprex/cpp/core/world.hpp

@@ -135,19 +135,11 @@ public:
     //
     void get_near(int64_t player_id, float radius, bool exclude_nowhere, bool omit_player, bool sorted, IdVector *into) const;
 
-    // get animation queue hashes.
+    // get encoded animation queues.
     //
-    // This is used by the graphics engine to check if any of the specified
+    // This is used by the graphics engine to get the animation queues.
-    // animation queues has changed in any way.
     //
-    void get_animation_queue_hashes(uint32_t count, const int64_t *ids, uint64_t *hashes);
+    void get_encoded_animation_queues(uint32_t count, const int64_t *ids, util::SharedStdStringVec &into);
-
-    // Get the animation queue for the graphics engine.
-    //
-    // Gets the animation queue of a tangible in a form that can be
-    // passed to the graphics engine.
-    //
-    void get_animation_queue(int64_t tanid, std::vector<EngineWrapper::AnimEntry> *into);
 
     // Make a tangible.
     //