Implement unicode on console, move readline into driver

luprex/Makefile

@@ -102,6 +102,7 @@ OBJ_DRV=\
 	obj/$(OS)/drv/driver.obj\
 	obj/$(OS)/drv/drvutil.obj\
 	obj/$(OS)/drv/sslutil.obj\
+	obj/$(OS)/drv/readline.obj\
 
 
 #######################################################################

luprex/cpp/core/drivenengine.cpp

@@ -69,108 +69,22 @@ static DrivenEngine *make_engine(std::string_view kind) {
 //////////////////////////////////////////////////////////////////////////////
 //////////////////////////////////////////////////////////////////////////////
 
-
 Channel::Channel(DrivenEngine *de, int chid, int port, const eng::string &target, bool stop) {
     chid_ = chid;
     port_ = port;
     closed_ = false;
     target_ = target;
-    readline_lastc_ = 0;
-    desired_prompt_ = "";
     stop_driver_ = stop;
     sb_in_ = eng::make_shared<StreamBuffer>();
     sb_out_ = eng::make_shared<StreamBuffer>();
-    sb_drvout_ = sb_out_;
-}
-
-void Channel::erase_command() {
-    int ccsize = current_prompt_.size() + current_command_.size();
-    if (ccsize > 0) {
-        sb_drvout_->write_bytes(util::repeat_string("\b \b", ccsize));
-        current_prompt_ = "";
-        current_command_ = "";
-    }
-}
-
-void Channel::echo_command() {
-    // If the prompt has changed, erase everything and start over.
-    if (desired_prompt_ != current_prompt_) {
-        int ccsize = current_prompt_.size() + current_command_.size();
-        sb_drvout_->write_bytes(util::repeat_string("\b \b", ccsize));
-        sb_drvout_->write_bytes(desired_prompt_);
-        current_command_ = "";
-        current_prompt_ = desired_prompt_;
-    }
-
-    // Find out how much of the command matches.
-    int match = sv::common_prefix_length(current_command_, desired_command_);
-
-    // Echo backspaces to remove the non-matching part.
-    int remove = current_command_.size() - match;
-    if (remove > 0) {
-        sb_drvout_->write_bytes(util::repeat_string("\b \b", remove));
-        current_command_ = current_command_.substr(0, match);
-    }
-
-    // Echo the new part.
-    eng::string newpart = desired_command_.substr(current_command_.size());
-    if (newpart != "") {
-        sb_drvout_->write_bytes(newpart);
-        current_command_ = desired_command_;
-    }
-}
-
-void Channel::set_prompt(const eng::string &p) {
-    desired_prompt_ = p;
-}
-
-void Channel::feed_readline(std::string_view data) {
-    int nbytes = data.size();
-    const char *bytes = data.data();
-    for (int i = 0; i < nbytes; i++) {
-        char c = bytes[i];
-        if ((c == '\n') && (readline_lastc_ == '\r')) {
-            // Ignore newline immediately after carriage return.
-            // Otherwise, crlf produces two newlines.
-        } else if ((c == '\r') || (c == '\n')) {
-            echo_command();
-            sb_drvout_->write_bytes(eng::string(" \n"));
-            sb_in_->write_bytes(desired_command_);
-            sb_in_->write_uint8('\n');
-            desired_command_ = "";
-            current_prompt_ = "";
-            current_command_ = "";
-        } else if ((c == '\b') || (c == 127)) {
-            int len = desired_command_.size();
-            if (len > 0) {
-                desired_command_ = desired_command_.substr(0, len-1);
-            }
-        } else if (c >= 32) {
-            int len = desired_command_.size();
-            if (len < READLINE_MAX) {
-                desired_command_ = desired_command_ + c;
-            }
-        }
-        readline_lastc_ = c;
-    }
 }
 
 std::string_view Channel::peek_outgoing() const {
-    return sb_drvout_->view();
+    return sb_out_->view();
-}
-
-void Channel::pump_readline() {
-    if (sb_drvout_ != sb_out_) {
-        if (!sb_out_->empty()) {
-            erase_command();
-            sb_out_->transfer_into(sb_drvout_.get());
-        }
-        echo_command();
-    }
 }
 
 void Channel::sent_outgoing(int nbytes) {
-    sb_drvout_->read_bytes(nbytes);
+    sb_out_->read_bytes(nbytes);
 }
 
 //////////////////////////////////////////////////////////////////////////////
@@ -232,7 +146,6 @@ void DrivenEngine::stop_driver() {
 DrivenEngine::DrivenEngine() {
     next_unused_chid_ = 1;
     stdio_channel_ = eng::make_shared<Channel>(this, 0, 0, "", false);
-    stdio_channel_->sb_drvout_ = eng::make_shared<StreamBuffer>();
     channels_[0] = stdio_channel_;
     rescan_lua_source_ = false;
     clock_ = 0.0;
@@ -492,7 +405,6 @@ bool DrivenEngine::drv_get_stop_driver() const {
 void DrivenEngine::drv_initialize(uint32_t srcpklen, const char *srcpk, int argc, char **argv) {
     drv_set_lua_source(srcpklen, srcpk);
     event_init(argc, argv);
-    stdio_channel_->pump_readline();
 }
 
 void DrivenEngine::drv_clear_new_outgoing() {
@@ -504,15 +416,11 @@ void DrivenEngine::drv_sent_outgoing(uint32_t chid, uint32_t nbytes) {
 }
 
 void DrivenEngine::drv_recv_incoming(uint32_t chid, uint32_t nbytes, const char *bytes) {
-    std::string_view sbytes(bytes, nbytes);
     if (nbytes > 0) {
+        std::string_view sbytes(bytes, nbytes);
         Channel *ch = get_chid(chid);
-        if (ch->sb_drvout_ != ch->sb_out_) {
-            ch->feed_readline(sbytes);
-        } else {
         ch->sb_in_->write_bytes(sbytes);
     }
-    }
 }
 
 void DrivenEngine::drv_notify_close(uint32_t chid, uint32_t len, const char *data) {
@@ -532,7 +440,6 @@ uint32_t DrivenEngine::drv_notify_accept(uint32_t port) {
 void DrivenEngine::drv_invoke_event_update(double clock) {
     clock_ = clock;
     event_update();
-    stdio_channel_->pump_readline();
 }
 
 void DrivenEngine::drv_set_lua_source(uint32_t srcpklen, const char *srcpk) {

luprex/cpp/core/drivenengine.hpp

@@ -88,10 +88,6 @@ public:
     //
     const eng::string &error() const { return error_; }
 
-    // Set the prompt for readline mode.
-    //
-    void set_prompt(const eng::string &prompt);
-
     // Do not construct your own Channels.  Instead,
     // use methods of class DrivenEngine like new_outgoing_channel.
     // Channels are referenced by shared_ptr.  You can
@@ -104,40 +100,22 @@ private:
     // Constructor is deliberately private.  Use
     // DrivenEngine::new_outgoing_channel to create outgoing socket channels.
     //
-
-    void feed_readline(std::string_view data);
     std::string_view peek_outgoing() const;
     void sent_outgoing(int nbytes);
-    void erase_command();
-    void echo_command();
-    void pump_readline();
 
 private:
-    static const int READLINE_MAX=512;
     int chid_;
 
     // These are the in/out buffers presented to the user.
     std::shared_ptr<StreamBuffer> sb_in_;
     std::shared_ptr<StreamBuffer> sb_out_;
 
-    // If this is stdio, we inject tty echoes into the output stream.
-    // This buffer holds the users output interleaved with the tty echoes.
-    // In any other channel, this is just another pointer to sb_out.
-    std::shared_ptr<StreamBuffer> sb_drvout_;
-
     int port_;
     bool closed_;
     eng::string error_;
     eng::string target_;
     bool stop_driver_;
 
-    // Readline stuff.  Only used on channel 0 (stdio).
-    eng::string desired_command_;
-    eng::string current_command_;
-    eng::string desired_prompt_;
-    eng::string current_prompt_;
-    char readline_lastc_;
-
     friend class DrivenEngine;
 };
 
@@ -215,9 +193,10 @@ public:
     // Obtain the stdio channel. There is only one stdio channel.
     //
     // DRIVER: the stdio channel is created automatically when the DrivenEngine
-    // is created.  The driver is responsible for relaying data into the channel
+    // is created.  Stdio should be connected to a console which is in
-    // using drv_get_target, drv_peek_outgoing, drv_sent_outgoing,
+    // line-at-a-time mode.  The driver is responsible for relaying data from
-    // drv_recv_incoming.
+    // the console into the stdio channel using drv_peek_outgoing,
+    // drv_sent_outgoing, drv_recv_incoming.
     //
     SharedChannel get_stdio_channel();
 

luprex/cpp/core/lpxclient.cpp

@@ -77,7 +77,7 @@ public:
         channel_ = new_outgoing_channel("nocert:localhost:8085");
 
         // Set the console prompt
-        get_stdio_channel()->set_prompt(console_.get_prompt());
+        // get_stdio_channel()->set_prompt(console_.get_prompt());
 
         // The driver loads the lua source automatically.
         // However, we don't need it.  Throw it out.
@@ -252,7 +252,7 @@ public:
             eng::string line = get_stdio_channel()->in()->readline();
             if (line == "") break;
             console_.add(line);
-            get_stdio_channel()->set_prompt(console_.get_prompt());
+            // get_stdio_channel()->set_prompt(console_.get_prompt());
             do_command(console_.get_command());
         }
 

luprex/cpp/core/lpxserver.cpp

@@ -51,7 +51,7 @@ public:
         listen_port(8080);
 
         // Set the console prompt.
-        get_stdio_channel()->set_prompt(console_.get_prompt());
+        // get_stdio_channel()->set_prompt(console_.get_prompt());
     }
 
     void do_luainvoke_command(const util::StringVec &words) {
@@ -172,7 +172,7 @@ public:
             eng::string line = get_stdio_channel()->in()->readline();
             if (line == "") break;
             console_.add(line);
-            get_stdio_channel()->set_prompt(console_.get_prompt());
+            // get_stdio_channel()->set_prompt(console_.get_prompt());
             do_command(console_.get_command());
         }
 

luprex/cpp/core/textgame.cpp

@@ -103,7 +103,7 @@ private:
         world_->run_unittests();
         actor_id_ = world_->create_login_actor();
         stdostream() << "Login actor ID: " << actor_id_ << std::endl;
-        get_stdio_channel()->set_prompt(console_.get_prompt());
+        //get_stdio_channel()->set_prompt(console_.get_prompt());
     }
 
     void event_update()
@@ -113,7 +113,7 @@ private:
             eng::string line = get_stdio_channel()->in()->readline();
             if (line == "") break;
             console_.add(line);
-            get_stdio_channel()->set_prompt(console_.get_prompt());
+            //get_stdio_channel()->set_prompt(console_.get_prompt());
             do_command(console_.get_command());
             if (print_channeler_.channel(world_->get_printbuffer(actor_id_), stdostream())) {
                 world_->invoke(print_channeler_.invocation(actor_id_));

luprex/cpp/core/util.cpp

@@ -301,6 +301,7 @@ int32_t read_ascii_char(string_view &source) {
 int32_t read_codepoint_utf8(string_view &source) {
     size_t size = source.size();
     if (size == 0) return -1;
+
     const unsigned char *bytes = (const unsigned char *)source.data();
     int codepoint;
     size_t seqlen;
@@ -321,7 +322,9 @@ int32_t read_codepoint_utf8(string_view &source) {
         codepoint = (bytes[0] & 0x07);
         seqlen = 4;
     } else {
-        return -1;
+        // Bad character. Drop a byte and return invalid CP.
+        source.remove_prefix(1);
+        return 1;
     }
 
     if (seqlen > size) {
@@ -329,7 +332,11 @@ int32_t read_codepoint_utf8(string_view &source) {
     }
 
     for (size_t i = 1; i < seqlen; ++i) {
-        if ((bytes[i] & 0xC0) != 0x80) return -1;
+        if ((bytes[i] & 0xC0) != 0x80) {
+            // Bad character. Drop a byte and return invalid CP.
+            source.remove_prefix(1);
+            return 1;
+        }
         codepoint = (codepoint << 6) | (bytes[i] & 0x3F);
     }
 
@@ -339,7 +346,9 @@ int32_t read_codepoint_utf8(string_view &source) {
         ((codepoint >= 0x0080) && (codepoint <= 0x07FF) && (seqlen != 2)) ||
         ((codepoint >= 0x0800) && (codepoint <= 0xFFFF) && (seqlen != 3)) ||
         ((codepoint >= 0x10000) && (codepoint <= 0x1FFFFF) && (seqlen != 4))) {
-        return -1;
+        // Bad character. Drop a byte and return invalid CP.
+        source.remove_prefix(1);
+        return 1;
     }
 
     source.remove_prefix(seqlen);

luprex/cpp/drv/driver-linux.cpp

@@ -2,6 +2,7 @@
 
 #include "drvutil.hpp"
 #include "sslutil.hpp"
+#include "readline.hpp"
 #include "../core/enginewrapper.hpp"
 
 #include <iostream>
@@ -217,12 +218,21 @@ static int socket_poll(struct pollfd *pollvec, int pollcount, int mstimeout, std
     return 0;
 }
 
-static int console_write(const char *bytes, int nbytes) {
+// Write unicode onto the console.
-    return write(1, bytes, nbytes);
+static void console_write(const CodepointString &cps) {
+    std::string utf8 = ReadlineDevice::to_utf8(cps);
+    write(1, utf8.c_str(), utf8.size());
 }
 
-static int console_read(char *bytes, int nbytes) {
+static CodepointString console_read() {
-    return read(0, bytes, nbytes);
+    CodepointString result;
+    char buffer[512];
+    int nread = read(0, buffer, 512);
+    if (nread > 0) {
+        std::string_view s(buffer, nread);
+        result = ReadlineDevice::from_utf8(s, nullptr);
+    }
+    return result;
 }
 
 static void call_init_engine_wrapper(const std::filesystem::path &luprexroot, EngineWrapper *w) {

luprex/cpp/drv/driver-windows.cpp

@@ -229,38 +229,51 @@ static void init_winsock() {
     }
 }
 
-static int console_write(const char *bytes, int nbytes) {
+static void console_write(const CodepointString &cps) {
-    if (nbytes == 0) return 0;
+    if (cps.size() == 0) return;
+    // Convert to wstring.
+    // Any character outside the range 0xFFFF is replaced with a box.
+    std::wstring ws(cps.size());
+    for (int i = 0; i < cps.size(); i++) {
+        char32_t c = cps[i];
+        if ((c >= 0)&&(c <= 0xFFFF)) ws[i] = (wchar_t)c;
+        else ws[i] = 0x2610;
+    }
     HANDLE hstdout = GetStdHandle(STD_OUTPUT_HANDLE);
     assert(hstdout != INVALID_HANDLE_VALUE);
     DWORD nwrote;
-    if (nbytes > 10000) nbytes = 10000;
+    std::wstring_view v(ws);
-    assert(WriteConsoleA(hstdout, bytes, nbytes, &nwrote, nullptr));
+    while (v.size() > 0) {
+        int nwrite = v.size();
+        if (nwrite > 10000) nwrite = 10000;
+        assert(WriteConsoleW(hstdout, v.data(), nwrite, &nwrote, nullptr));
         assert(nwrote > 0);
-    return nwrote;
+        v.remove_prefix(nwrote);
+    }
 }
 
-static int console_read(char *bytes, int nbytes) {
+static CodepointString console_read() {
     HANDLE hstdin = GetStdHandle(STD_INPUT_HANDLE);
     assert(hstdin != INVALID_HANDLE_VALUE);
     INPUT_RECORD inrecords[512];
     DWORD nread, nevents;
-    int nascii = 0;
     if (GetNumberOfConsoleInputEvents(hstdin, &nevents)) {
-        if (int(nevents) > nbytes) nevents = nbytes;
+        if (int(nevents) > 0) {
-        ReadConsoleInputA(hstdin, inrecords, nevents, &nread);
+            if (int(nevents) > 512) nevents = 512;
+            ReadConsoleInputW(hstdin, inrecords, nevents, &nread);
+            CodepointString result(nread, 0);
+            int len = 0;
             for (int i = 0; i < int(nread); i++) {
                 const INPUT_RECORD &inr = inrecords[i];
                 if (inr.EventType != KEY_EVENT) continue;
                 const KEY_EVENT_RECORD &key = inr.Event.KeyEvent;
                 if (!key.bKeyDown) continue;
-            char c = key.uChar.AsciiChar;
+                result[len++] = key.uChar.UnicodeChar;
-            bytes[nascii++] = c;
             }
-        return nascii;
+            return result.substr(0, len);
-    } else {
-        return 0;
         }
+    }
+    return CodepointString();
 }
 
 static void ssl_load_certificate_authorities(SSL_CTX *ctx) {

luprex/cpp/drv/driver.cpp

@@ -54,6 +54,7 @@ class Driver {
         CHAN_SSL_ACCEPTING,
         CHAN_SSL_READWRITE,
     };
+
     struct ChanInfo {
         int chid;
         SOCKET socket;
@@ -89,6 +90,7 @@ class Driver {
     bool read_console_recently_;
     std::unique_ptr<struct pollfd[]> pollvec_;
     std::unique_ptr<char[]> chbuf_;
+    ReadlineDevice readline_device_;
 
     sslutil::UniqueCTX ssl_server_ctx_;
     sslutil::UniqueCTX ssl_client_secure_ctx_;
@@ -202,20 +204,27 @@ class Driver {
             engw.get_outgoing(&engw, 0, &ndata, &data);
             if (ndata == 0) break;
             if (ndata > DRV_SHORTSTRING_SIZE) ndata = DRV_SHORTSTRING_SIZE;
-            int nwrote = console_write(data, ndata);
+            std::string_view src(data, ndata);
-            if (nwrote <= 0) break;
+            int consumed;
-            engw.play_sent_outgoing(&engw, 0, nwrote);
+            CodepointString cps = ReadlineDevice::from_utf8(src, &consumed);
+            readline_device_.print(cps);
+            engw.play_sent_outgoing(&engw, 0, consumed);
         }
     }
 
     void handle_console_input() {
-        char buffer[256];
         read_console_recently_ = false;
         while (true) {
-            int nread = console_read(buffer, 256);
+            CodepointString cps = console_read();
-            if (nread <= 0) break;
+            if (cps.size() == 0) break;
             read_console_recently_ = true;
-            engw.play_recv_incoming(&engw, 0, nread, buffer);
+            for (char32_t c : cps) {
+                CodepointString line = readline_device_.putcode(c);
+                if (!line.empty()) {
+                    std::string utf8 = ReadlineDevice::to_utf8(line);
+                    engw.play_recv_incoming(&engw, 0, utf8.size(), utf8.c_str());
+                }
+            }
         }
     }
 
@@ -575,6 +584,9 @@ class Driver {
     }
 
     int drive(int argc, char *argv[]) {
+        // Set up the console readline device.
+        readline_device_.set_print_callback(console_write);
+
         // Remove the program name from argv.
         std::string program = argv[0];
         argc -= 1;

luprex/cpp/drv/readline.cpp

@@ -0,0 +1,225 @@
+#include "readline.hpp"
+
+#define MAXLINE 512
+
+static CodepointString n_backspaces(int n) {
+    CodepointString result(3 * n, 0);
+    for (int i = 0; i < n; i++) {
+        result[i*3 + 0] = '\b';
+        result[i*3 + 1] = ' ';
+        result[i*3 + 2] = '\b';
+    }
+    return result;
+}
+
+static int common_prefix_length(const CodepointString &a, const CodepointString &b) {
+    int minlen = std::min(a.size(), b.size());
+    for (int i = 0; i < minlen; i++) {
+        if (a[i] != b[i]) return i;
+    }
+    return minlen;
+}
+
+static int buffer_codepoint_utf8(char32_t scp, char *buffer) {
+    uint32_t cp = (uint32_t)scp;
+    unsigned char *c = (unsigned char *)buffer;
+    if (cp < 0) {
+        return 0;
+    }
+    else if (cp <= 0x7F) {
+        c[0] = cp;
+        return 1;
+    }
+    else if (cp <= 0x7FF) {
+        c[0] = (cp>>6)+192;
+        c[1] = (cp&63)+128;
+        return 2;
+    }
+    else if (cp <= 0xFFFF) {
+        c[0] = (cp>>12)+224;
+        c[1] = ((cp>>6)&63)+128;
+        c[2] = (cp&63)+128;
+        return 3;
+    }
+    else if (cp <= 0x10FFFF) {
+        c[0] = (cp>>18)+240;
+        c[1] = ((cp>>12)&63)+128;
+        c[2] = ((cp>>6)&63)+128;
+        c[3] = (cp&63)+128;
+        return 4;
+    } else {
+        return 0;
+    }
+}
+
+static int32_t read_codepoint_utf8(std::string_view &source) {
+    size_t size = source.size();
+    if (size == 0) return -1;
+
+    const unsigned char *bytes = (const unsigned char *)source.data();
+    int codepoint;
+    size_t seqlen;
+    if ((bytes[0] & 0x80) == 0x00) {
+        // U+0000 to U+007F
+        codepoint = (bytes[0] & 0x7F);
+        seqlen = 1;
+    } else if ((bytes[0] & 0xE0) == 0xC0) {
+        // U+0080 to U+07FF
+        codepoint = (bytes[0] & 0x1F);
+        seqlen = 2;
+    } else if ((bytes[0] & 0xF0) == 0xE0) {
+        // U+0800 to U+FFFF
+        codepoint = (bytes[0] & 0x0F);
+        seqlen = 3;
+    } else if ((bytes[0] & 0xF8) == 0xF0) {
+        // U+10000 to U+10FFFF
+        codepoint = (bytes[0] & 0x07);
+        seqlen = 4;
+    } else {
+        // Bad character. Drop a byte and return invalid CP.
+        source.remove_prefix(1);
+        return -2;
+    }
+
+    if (seqlen > size) {
+        return -1;
+    }
+
+    for (size_t i = 1; i < seqlen; ++i) {
+        if ((bytes[i] & 0xC0) != 0x80) {
+            // Bad character. Drop a byte and return invalid CP.
+            source.remove_prefix(1);
+            return -2;
+        }
+        codepoint = (codepoint << 6) | (bytes[i] & 0x3F);
+    }
+
+    if ((codepoint > 0x10FFFF) ||
+        ((codepoint <= 0x007F) && (seqlen != 1)) ||
+        ((codepoint >= 0x0080) && (codepoint <= 0x07FF) && (seqlen != 2)) ||
+        ((codepoint >= 0x0800) && (codepoint <= 0xFFFF) && (seqlen != 3)) ||
+        ((codepoint >= 0x10000) && (codepoint <= 0x1FFFFF) && (seqlen != 4))) {
+        // Bad character. Drop a byte and return invalid CP.
+        source.remove_prefix(1);
+        return -2;
+    }
+
+    source.remove_prefix(seqlen);
+    return codepoint;
+}
+
+ReadlineDevice::ReadlineDevice() {
+    desired_prompt_ = CodepointString(1, '>');
+}
+
+
+void ReadlineDevice::set_print_callback(print_callback cb) {
+    print_cb_ = cb;
+}
+
+void ReadlineDevice::erase_command() {
+    int ccsize = current_prompt_.size() + current_command_.size();
+    if (ccsize > 0) {
+        print_cb_(n_backspaces(ccsize));
+        current_prompt_.clear();
+        current_command_.clear();
+    }
+}
+
+void ReadlineDevice::echo_command() {
+    // If the prompt has changed, erase everything and start over.
+    if (desired_prompt_ != current_prompt_) {
+        int ccsize = current_prompt_.size() + current_command_.size();
+        print_cb_(n_backspaces(ccsize));
+        print_cb_(desired_prompt_);
+        current_command_.clear();
+        current_prompt_ = desired_prompt_;
+    }
+
+    // Find out how much of the command matches.
+    int match = common_prefix_length(current_command_, desired_command_);
+
+    // Echo backspaces to remove the non-matching part.
+    int remove = current_command_.size() - match;
+    if (remove > 0) {
+        print_cb_(n_backspaces(remove));
+        current_command_ = current_command_.substr(0, match);
+    }
+
+    // Echo the new part.
+    CodepointString newpart = desired_command_.substr(current_command_.size());
+    if (!newpart.empty()) {
+        print_cb_(newpart);
+        current_command_ = desired_command_;
+    }
+}
+
+CodepointString ReadlineDevice::putcode(char32_t c) {
+    if ((c == '\n') && (readline_lastc_ == '\r')) {
+        // Ignore newline immediately after carriage return.
+        // Otherwise, crlf produces two newlines.
+        return CodepointString();
+    } else if ((c == '\r') || (c == '\n')) {
+        CodepointString white(1, ' ');
+        CodepointString newline(1, '\n');
+        echo_command();
+        print_cb_(white + newline);
+        CodepointString result = desired_command_ + newline;
+        desired_command_.clear();
+        current_prompt_.clear();
+        current_command_.clear();
+        return result;
+    } else if ((c == '\b') || (c == 127)) {
+        int len = desired_command_.size();
+        if (len > 0) {
+            desired_command_ = desired_command_.substr(0, len-1);
+        }
+        echo_command();
+        return CodepointString();
+    } else if ((c >= 32)&&(c <= 0x10FFFF)) {
+        int len = desired_command_.size();
+        if (len < MAXLINE) {
+            desired_command_ = desired_command_ + c;
+        }
+        echo_command();
+        return CodepointString();
+    }
+    readline_lastc_ = c;
+    return CodepointString();
+}
+
+void ReadlineDevice::print(const CodepointString &s) {
+    if (!s.empty()) {
+        erase_command();
+        print_cb_(s);
+        echo_command();
+    }
+}
+
+std::string ReadlineDevice::to_utf8(const CodepointString &s) {
+    std::string result(s.size() * 4, 0);
+    char *buffer = &result[0];
+    int len = 0;
+    for (char32_t c : s) {
+        int clen = buffer_codepoint_utf8(c, buffer + len);
+        len += clen;
+    }
+    return result.substr(0, len);
+}
+
+CodepointString ReadlineDevice::from_utf8(std::string_view s, int *consumed) {
+    std::string_view rest = s;
+    CodepointString result(s.size(), 0);
+    int len = 0;
+    while (true) {
+        int32_t c = read_codepoint_utf8(rest);
+        if (c == -1) break; // EOF reached;
+        if (c == -2) continue; // Filter out bad UTF8 but continue.
+        result[len++] = (char32_t)c;
+    }
+    if (consumed != nullptr) {
+        *consumed = s.size() - rest.size();
+    }
+    return result.substr(0, len);
+}
+

luprex/cpp/drv/readline.hpp

@@ -0,0 +1,53 @@
+
+#ifndef READLINE_HPP
+#define READLINE_HPP
+
+#include <string>
+#include <string_view>
+
+using CodepointString = std::basic_string<char32_t>;
+
+class ReadlineDevice {
+public:
+    using print_callback = void (*)(const CodepointString &text);
+
+private:
+    print_callback print_cb_;
+    CodepointString desired_command_;
+    CodepointString current_command_;
+    CodepointString desired_prompt_;
+    CodepointString current_prompt_;
+    char32_t readline_lastc_;
+
+    void echo_command();
+    void erase_command();
+
+public:
+    ReadlineDevice();
+
+    // The callback must be set before using the readline device.
+    void set_print_callback(print_callback cb);
+    
+    // change the prompt.
+    void set_prompt(const CodepointString &prompt);
+
+    // Use this to print anything on the console.
+    void print(const CodepointString &cps);
+
+    // Whenever the user types a character, call 'putcode'. If the code is
+    // newline, this returns the line of text that was entered, including the
+    // newline.  Otherwise returns empty string. Backspace is handled here.
+    CodepointString putcode(char32_t codepoint);
+
+    // This can be used to convert a codepoint string into a
+    // UTF8-string.
+    static std::string to_utf8(const CodepointString &cps);
+
+    // This can be used to convert UTF8 to a codepoint string.
+    // Some of the bytes may not be consumed.  Returns the Codepoint
+    // string and the number of bytes consumed.
+    static CodepointString from_utf8(std::string_view source, int *consumed);
+};
+
+
+#endif // READLINE_HPP