Add the mcufont encoder to the tools (including a win32 build) with binaries

3rdparty/freetype-2.6.1/readme.txt

@@ -0,0 +1,2 @@
+freetype-2.6.1 source code has not been included in this repository so that you are fully
+aware of the license types and restrictions before using it with uGFX.

tools/mcufontencoder/src/Makefile

@@ -0,0 +1,43 @@
+CXXFLAGS = -O2 -Wall -Werror -Wno-unused-function -Wno-sign-compare -std=c++0x
+CXXFLAGS += -ggdb
+LDFLAGS += -pthread
+
+# Libfreetype
+CXXFLAGS += $(shell freetype-config --cflags)
+LDFLAGS += $(shell freetype-config --libs)
+
+# Class to represent font data internally
+OBJS = datafile.o
+
+# Utility functions
+OBJS += importtools.o exporttools.o
+
+# Import formats
+OBJS += bdf_import.o freetype_import.o
+
+# rlefont export format
+OBJS += encode_rlefont.o optimize_rlefont.o export_rlefont.o
+
+# bwfont export format
+OBJS += export_bwfont.o
+
+
+all: run_unittests mcufont
+
+clean:
+	rm -f unittests unittests.cc mcufont $(OBJS)
+
+mcufont: main.o $(OBJS)
+	g++ $(CXXFLAGS) -o $@ $^ $(LDFLAGS)
+
+unittests.cc: *.hh
+	cxxtestgen --have-eh --error-printer -o unittests.cc $^
+
+unittests: unittests.cc $(OBJS)
+	g++ $(CXXFLAGS) -o $@ $^ $(LDFLAGS)
+
+run_unittests: unittests
+	./unittests
+
+%.o: %.cc *.hh
+	g++ $(CXXFLAGS) -c $<

tools/mcufontencoder/src/Makefile.mingw32

@@ -0,0 +1,44 @@
+ARCH     = i686-pc-mingw32-
+CXXFLAGS = -O2 -Wall -Werror -Wno-unused-function -Wno-sign-compare -std=c++0x
+CXXFLAGS += -ggdb
+LDFLAGS += -pthread --static
+
+# Libfreetype
+CXXFLAGS += $(shell freetype-config --cflags)
+LDFLAGS += $(shell freetype-config --libs)
+#FREETYPE2_LIB = ../ugfx/3rdparty/freetype-2.6.1
+#CXXFLAGS += -I$(FREETYPE2_LIB)/include
+#LDFLAGS += -I$(FREETYPE2_LIB)/lib -lfreetype
+
+# compiler fixes for mingw32
+CXXFLAGS += -DNEED_STRING_FIXES -DNEED_THREAD_FIXES
+
+# Class to represent font data internally
+OBJS = datafile.o
+
+# Utility functions
+OBJS += importtools.o exporttools.o
+
+# Import formats
+OBJS += bdf_import.o freetype_import.o
+
+# rlefont export format
+OBJS += encode_rlefont.o optimize_rlefont.o export_rlefont.o
+
+# bwfont export format
+OBJS += export_bwfont.o
+
+
+all: mcufont
+
+strip: mcufont.exe
+	strip mcufont.exe
+
+clean:
+	rm -f mcufont $(OBJS)
+
+mcufont: main.o $(OBJS)
+	$(ARCH)g++ $(CXXFLAGS) -o $@ $^ $(LDFLAGS)
+
+%.o: %.cc *.hh
+	$(ARCH)g++ $(CXXFLAGS) -c $<

tools/mcufontencoder/src/bdf_import.cc

@@ -0,0 +1,156 @@
+#include "bdf_import.hh"
+#include "importtools.hh"
+#include <sstream>
+#include <string>
+#include <cctype>
+#include <stdexcept>
+
+namespace mcufont {
+
+static std::string toupper(const std::string &input)
+{
+    std::string result;
+    for (char c: input) result.push_back(::toupper(c));
+    return result;
+}
+
+static int hextoint(char c)
+{
+    if (c >= '0' && c <= '9') return c - '0';
+    if (c >= 'A' && c <= 'F') return c - 'A' + 10;
+    throw std::domain_error("Hex digit not in range");
+}
+
+static void parse_fontinfo(std::istream &file, DataFile::fontinfo_t &fontinfo)
+{
+    std::string line;
+    while (std::getline(file, line))
+    {
+        std::istringstream s(line);
+        std::string tag;
+        s >> tag;
+        tag = toupper(tag);
+        
+        if (tag == "FONT")
+        {
+            while (isspace(s.peek())) s.get();
+            std::getline(s, fontinfo.name);
+        }
+        else if (tag == "FONTBOUNDINGBOX")
+        {
+            int x, y;
+            s >> fontinfo.max_width >> fontinfo.max_height;
+            s >> x >> y;
+            fontinfo.baseline_x = - x;
+            fontinfo.baseline_y = fontinfo.max_height + y;
+        }
+        else if (tag == "STARTCHAR")
+        {
+            break;
+        }
+    }
+}
+
+static bool parse_glyph(std::istream &file, DataFile::glyphentry_t &glyph,
+                        const DataFile::fontinfo_t &fontinfo)
+{
+    glyph.chars.clear();
+    glyph.width = 0;
+    
+    // Initialize the character contents to all 0 with proper size.
+    glyph.data.clear();
+    glyph.data.resize(fontinfo.max_width * fontinfo.max_height, 0);
+    
+    int bbx_w = fontinfo.max_width;
+    int bbx_h = fontinfo.max_height;
+    int bbx_x = - fontinfo.baseline_x;
+    int bbx_y = fontinfo.baseline_y - fontinfo.max_height;
+    
+    // Read glyph metadata
+    std::string line;
+    std::string tag;
+    while (std::getline(file, line))
+    {
+        std::istringstream s(line);
+        s >> tag;
+        tag = toupper(tag);
+        
+        if (tag == "ENCODING")
+        {
+            int c;
+            s >> c;
+            glyph.chars.push_back(c);
+        }
+        else if (tag == "DWIDTH")
+        {
+            s >> glyph.width;
+        }
+        else if (tag == "BBX")
+        {
+            s >> bbx_w >> bbx_h >> bbx_x >> bbx_y;
+        }
+        else if (tag == "BITMAP")
+        {
+            break;
+        }
+    }
+    
+    if (tag != "BITMAP")
+        return false;
+    
+    // Read glyph bits
+    int x0 = fontinfo.baseline_x + bbx_x;
+    int y = fontinfo.baseline_y - bbx_y - bbx_h;
+    for (int i = 0; i < bbx_h; i++)
+    {
+        std::getline(file, line);
+        line = toupper(line);
+        
+        for (int x = 0; x < bbx_w; x++)
+        {
+            int nibble = hextoint(line.at(x / 4));
+            uint8_t pixel = 0;
+            if (nibble & (8 >> (x % 4)))
+                pixel = 15;
+            
+            glyph.data.at(y * fontinfo.max_width + x0 + x) = pixel;
+        }
+        
+        y++;
+    }
+    
+    std::getline(file, line);
+    line = toupper(line);
+    if (line.compare(0, 7, "ENDCHAR") == 0)
+        return true;
+    else
+        return false;
+}
+
+std::unique_ptr<DataFile> LoadBDF(std::istream &file)
+{
+    DataFile::fontinfo_t fontinfo = {};
+    std::vector<DataFile::glyphentry_t> glyphtable;
+    std::vector<DataFile::dictentry_t> dictionary;
+   
+    parse_fontinfo(file, fontinfo);
+    
+    while (file)
+    {
+        DataFile::glyphentry_t glyph = {};
+        if (parse_glyph(file, glyph, fontinfo))
+            glyphtable.push_back(glyph);
+    }
+    
+    eliminate_duplicates(glyphtable);
+    crop_glyphs(glyphtable, fontinfo);
+    detect_flags(glyphtable, fontinfo);
+    
+    fontinfo.line_height = fontinfo.max_height;
+    
+    std::unique_ptr<DataFile> result(new DataFile(
+        dictionary, glyphtable, fontinfo));
+    return result;
+}
+
+}

tools/mcufontencoder/src/bdf_import.hh

@@ -0,0 +1,80 @@
+// Function for importing .BDF fonts as data files.
+
+#pragma once
+#include "datafile.hh"
+
+namespace mcufont
+{
+
+std::unique_ptr<DataFile> LoadBDF(std::istream &file);
+
+}
+
+#ifdef CXXTEST_RUNNING
+#include <cxxtest/TestSuite.h>
+
+using namespace mcufont;
+
+class BDFTests: public CxxTest::TestSuite
+{
+public:
+    void testLoadBDF()
+    {
+        std::istringstream s(testfile);
+        std::unique_ptr<DataFile> f = LoadBDF(s);
+        
+        TS_ASSERT_EQUALS(f->GetFontInfo().name, "-Misc-Fixed-Medium-R-Normal--14-130-75-75-C-70-ISO8859-15");
+        TS_ASSERT_EQUALS(f->GetFontInfo().max_width, 6);
+        TS_ASSERT_EQUALS(f->GetFontInfo().max_height, 11);
+        TS_ASSERT_EQUALS(f->GetGlyphCount(), 1);
+        TS_ASSERT_EQUALS(f->GetGlyphEntry(0).chars.size(), 2);
+    }
+    
+private:
+    static constexpr const char *testfile = 
+        "STARTFONT 2.1\n"
+        "FONT -Misc-Fixed-Medium-R-Normal--14-130-75-75-C-70-ISO8859-15\n"
+        "FONTBOUNDINGBOX 7 14 0 -2\n"
+        "STARTCHAR defaultchar\n"
+        "ENCODING 0\n"
+        "DWIDTH 7 0\n"
+        "BBX 7 14 0 -2\n"
+        "BITMAP\n"
+        "00\n"
+        "B4\n"
+        "84\n"
+        "00\n"
+        "84\n"
+        "84\n"
+        "00\n"
+        "84\n"
+        "84\n"
+        "00\n"
+        "84\n"
+        "B4\n"
+        "00\n"
+        "00\n"
+        "ENDCHAR\n"
+        "STARTCHAR copychar\n"
+        "ENCODING 2\n"
+        "DWIDTH 7 0\n"
+        "BBX 7 14 0 -2\n"
+        "BITMAP\n"
+        "00\n"
+        "B4\n"
+        "84\n"
+        "00\n"
+        "84\n"
+        "84\n"
+        "00\n"
+        "84\n"
+        "84\n"
+        "00\n"
+        "84\n"
+        "B4\n"
+        "00\n"
+        "00\n"
+        "ENDCHAR\n";
+
+};
+#endif

tools/mcufontencoder/src/ccfixes.hh

@@ -0,0 +1,152 @@
+
+#ifdef NEED_STRING_FIXES
+	#include <string>
+	#include <sstream>
+	#include <stdexcept>
+	#include <limits>
+	#include <cstdlib>
+
+	namespace std {
+		template <typename T> inline std::string to_string(T value)
+		{
+			std::ostringstream os ;
+			os << value ;
+			return os.str() ;
+		}
+
+		inline int stoi( const std::string& str, std::size_t* pos = 0, int base = 10 )
+		{
+			const char* begin = str.c_str() ;
+			char* end = nullptr ;
+			long value = std::strtol( begin, &end, base ) ;
+
+			if( errno == ERANGE || value > std::numeric_limits<int>::max() )
+				throw std::out_of_range( "stoi: out ofrange" ) ;
+
+			if( end == str.c_str() )
+				throw std::invalid_argument( "stoi: invalid argument" ) ;
+
+			if(pos) *pos = end - begin ;
+
+			return value ;
+		}
+	}
+#endif
+
+#ifdef NEED_THREAD_FIXES
+#ifndef WIN32STDTHREAD_H
+	#define WIN32STDTHREAD_H
+
+	#include <windows.h>
+	#include <functional>
+	#include <memory>
+	#include <chrono>
+	#include <system_error>
+	#include <process.h>
+
+	#define _STD_THREAD_INVALID_HANDLE 0
+	namespace std
+	{
+
+
+		class thread
+		{
+		public:
+			class id
+			{
+				DWORD mId;
+				void clear() {mId = 0;}
+				friend class thread;
+			public:
+				id(DWORD aId=0):mId(aId){}
+				bool operator==(const id& other) const {return mId == other.mId;}
+			};
+		protected:
+			HANDLE mHandle;
+			id mThreadId;
+		public:
+			typedef HANDLE native_handle_type;
+			id get_id() const noexcept {return mThreadId;}
+			native_handle_type native_handle() const {return mHandle;}
+			thread(): mHandle(_STD_THREAD_INVALID_HANDLE){}
+			thread(thread& other)
+			:mHandle(other.mHandle), mThreadId(other.mThreadId)
+			{
+				other.mHandle = _STD_THREAD_INVALID_HANDLE;
+				other.mThreadId.clear();
+			}
+			template<class Function, class... Args>
+			explicit thread(Function&& f, Args&&... args)
+			{
+				typedef decltype(std::bind(f, args...)) Call;
+				Call* call = new Call(std::bind(f, args...));
+				mHandle = (HANDLE)_beginthreadex(NULL, 0, threadfunc<Call>,
+					(LPVOID)call, 0, (unsigned*)&(mThreadId.mId));
+			}
+			template <class Call>
+			static unsigned int __stdcall threadfunc(void* arg)
+			{
+				std::unique_ptr<Call> upCall(static_cast<Call*>(arg));
+				(*upCall)();
+				return (unsigned long)0;
+			}
+			bool joinable() const {return mHandle != _STD_THREAD_INVALID_HANDLE;}
+			void join()
+			{
+				if (get_id() == GetCurrentThreadId())
+					throw system_error(EDEADLK, generic_category());
+				if (mHandle == _STD_THREAD_INVALID_HANDLE)
+					throw system_error(ESRCH, generic_category());
+				if (!joinable())
+					throw system_error(EINVAL, generic_category());
+				WaitForSingleObject(mHandle, INFINITE);
+				CloseHandle(mHandle);
+				mHandle = _STD_THREAD_INVALID_HANDLE;
+				mThreadId.clear();
+			}
+
+			~thread()
+			{
+				if (joinable())
+					std::terminate();
+			}
+			thread& operator=(const thread&) = delete;
+			thread& operator=(thread&& other) noexcept
+			{
+				if (joinable())
+				  std::terminate();
+				swap(std::forward<thread>(other));
+				return *this;
+			}
+			void swap(thread&& other) noexcept
+			{
+				std::swap(mHandle, other.mHandle);
+				std::swap(mThreadId.mId, other.mThreadId.mId);
+			}
+			static unsigned int hardware_concurrency() noexcept {return 1;}
+			void detach()
+			{
+				if (!joinable())
+					throw system_error();
+				mHandle = _STD_THREAD_INVALID_HANDLE;
+				mThreadId.clear();
+			}
+		};
+		namespace this_thread
+		{
+			inline thread::id get_id() {return thread::id(GetCurrentThreadId());}
+			inline void yield() {Sleep(0);}
+			template< class Rep, class Period >
+			void sleep_for( const std::chrono::duration<Rep,Period>& sleep_duration)
+			{
+				Sleep(chrono::duration_cast<chrono::milliseconds>(sleep_duration).count());
+			}
+			template <class Clock, class Duration>
+			void sleep_until(const std::chrono::time_point<Clock,Duration>& sleep_time)
+			{
+				sleep_for(sleep_time-Clock::now());
+		   }
+		}
+	}
+#endif
+#endif

tools/mcufontencoder/src/datafile.cc

@@ -0,0 +1,239 @@
+#include "datafile.hh"
+#include <sstream>
+#include <algorithm>
+#include <cctype>
+#include <stdexcept>
+#include "ccfixes.hh"
+
+#define DATAFILE_FORMAT_VERSION 1
+
+namespace mcufont {
+
+DataFile::DataFile(const std::vector<dictentry_t> &dictionary,
+                   const std::vector<glyphentry_t> &glyphs,
+                   const fontinfo_t &fontinfo):
+    m_dictionary(dictionary), m_glyphtable(glyphs), m_fontinfo(fontinfo)
+{
+    dictentry_t dummy = {};
+    while (m_dictionary.size() < dictionarysize)
+        m_dictionary.push_back(dummy);
+    
+    UpdateLowScoreIndex();
+}
+
+void DataFile::Save(std::ostream &file) const
+{
+    file << "Version " << DATAFILE_FORMAT_VERSION << std::endl;
+    file << "FontName " << m_fontinfo.name << std::endl;
+    file << "MaxWidth " << m_fontinfo.max_width << std::endl;
+    file << "MaxHeight " << m_fontinfo.max_height << std::endl;
+    file << "BaselineX " << m_fontinfo.baseline_x << std::endl;
+    file << "BaselineY " << m_fontinfo.baseline_y << std::endl;
+    file << "LineHeight " << m_fontinfo.line_height << std::endl;
+    file << "Flags " << m_fontinfo.flags << std::endl;
+    file << "RandomSeed " << m_seed << std::endl;
+    
+    for (const dictentry_t &d : m_dictionary)
+    {
+        if (d.replacement.size() != 0)
+        {
+            file << "DictEntry " << d.score << " ";
+            file << d.ref_encode << " " << d.replacement << std::endl;
+        }
+    }
+    
+    for (const glyphentry_t &g : m_glyphtable)
+    {
+        file << "Glyph ";
+        for (size_t i = 0; i < g.chars.size(); i++)
+        {
+            if (i != 0) file << ',';
+            file << g.chars.at(i);
+        }
+        file << " " << g.width << " " << g.data << std::endl;
+    }
+}
+
+std::unique_ptr<DataFile> DataFile::Load(std::istream &file)
+{
+    fontinfo_t fontinfo = {};
+    std::vector<dictentry_t> dictionary;
+    std::vector<glyphentry_t> glyphtable;
+    uint32_t seed = 1234;
+    int version = -1;
+    
+    std::string line;
+    while (std::getline(file, line))
+    {
+        std::istringstream input(line);
+        std::string tag;
+        
+        input >> tag;
+        
+        if (tag == "Version")
+        {
+            input >> version;
+        }
+        else if (tag == "FontName")
+        {
+            while (std::isspace(input.peek())) input.get();
+            std::getline(input, fontinfo.name);
+        }
+        else if (tag == "MaxWidth")
+        {
+            input >> fontinfo.max_width;
+        }
+        else if (tag == "MaxHeight")
+        {
+            input >> fontinfo.max_height;
+        }
+        else if (tag == "BaselineX")
+        {
+            input >> fontinfo.baseline_x;
+        }
+        else if (tag == "BaselineY")
+        {
+            input >> fontinfo.baseline_y;
+        }
+        else if (tag == "LineHeight")
+        {
+            input >> fontinfo.line_height;
+        }
+        else if (tag == "RandomSeed")
+        {
+            input >> seed;
+        }
+        else if (tag == "Flags")
+        {
+            input >> fontinfo.flags;
+        }
+        else if (tag == "DictEntry" && dictionary.size() < dictionarysize)
+        {
+            dictentry_t d = {};
+            input >> d.score >> d.ref_encode >> d.replacement;
+            dictionary.push_back(d);
+        }
+        else if (tag == "Glyph")
+        {
+            glyphentry_t g = {};
+            std::string chars;
+            input >> chars >> g.width >> g.data;
+            
+            if ((int)g.data.size() != fontinfo.max_width * fontinfo.max_height)
+                throw std::runtime_error("wrong glyph data length: " + std::to_string(g.data.size()));
+            
+            size_t pos = 0;
+            while (pos < chars.size()) {
+                size_t p;
+                g.chars.push_back(std::stoi(chars.substr(pos), &p));
+                pos += p + 1;
+            }
+            
+            glyphtable.push_back(g);
+        }
+    }
+    
+    if (version != DATAFILE_FORMAT_VERSION)
+    {
+        return std::unique_ptr<DataFile>(nullptr);
+    }
+    
+    std::unique_ptr<DataFile> result(new DataFile(dictionary, glyphtable, fontinfo));
+    result->SetSeed(seed);
+    return result;
+}
+
+void DataFile::SetDictionaryEntry(size_t index, const dictentry_t &value)
+{
+    m_dictionary.at(index) = value;
+    
+    if (index == m_lowscoreindex ||
+        m_dictionary.at(m_lowscoreindex).score > value.score)
+    {
+        UpdateLowScoreIndex();
+    }
+}
+
+std::map<size_t, size_t> DataFile::GetCharToGlyphMap() const
+{
+    std::map<size_t, size_t> char_to_glyph;
+    
+    for (size_t i = 0; i < m_glyphtable.size(); i++)
+    {
+        for (size_t c: m_glyphtable[i].chars)
+        {
+            char_to_glyph[c] = i;
+        }
+    }
+    
+    return char_to_glyph;
+}
+
+std::string DataFile::GlyphToText(size_t index) const
+{
+    std::ostringstream os;
+    
+    const char glyphchars[] = "....,,,,----XXXX";
+    
+    for (int y = 0; y < m_fontinfo.max_height; y++)
+    {
+        for (int x = 0; x < m_fontinfo.max_width; x++)
+        {
+            size_t pos = y * m_fontinfo.max_width + x;
+            os << glyphchars[m_glyphtable.at(index).data.at(pos)];
+        }
+        os << std::endl;
+    }
+    
+    return os.str();
+}
+
+void DataFile::UpdateLowScoreIndex()
+{
+    auto comparison = [](const dictentry_t &a, const dictentry_t &b)
+    {
+        return a.score < b.score;
+    };
+    
+    auto iter = std::min_element(m_dictionary.begin(),
+                                 m_dictionary.end(),
+                                 comparison);
+    
+    m_lowscoreindex = iter - m_dictionary.begin();
+}
+
+std::ostream& operator<<(std::ostream& os, const DataFile::pixels_t& str)
+{
+    for (uint8_t p: str)
+    {
+        if (p <= 9)
+            os << (char)(p + '0');
+        else if (p <= 15)
+            os << (char)(p - 10 + 'A');
+        else
+            throw std::logic_error("invalid pixel alpha: " + std::to_string(p));
+    }
+    return os;
+}
+
+std::istream& operator>>(std::istream& is, DataFile::pixels_t& str)
+{
+    char c;
+    str.clear();
+    
+    while (isspace(is.peek())) is.get();
+    
+    while (is.get(c))
+    {
+        if (c >= '0' && c <= '9')
+            str.push_back(c - '0');
+        else if (c >= 'A' && c <= 'F')
+            str.push_back(c - 'A' + 10);
+        else
+            break;
+    }
+    
+    return is;
+}
+
+}

tools/mcufontencoder/src/datafile.hh

@@ -0,0 +1,174 @@
+// Class to store the data of a font while it is being processed.
+// This class can be safely cloned using the default copy constructor.
+
+#pragma once
+#include <cstdint>
+#include <vector>
+#include <string>
+#include <fstream>
+#include <memory>
+#include <map>
+
+namespace mcufont
+{
+
+class DataFile
+{
+public:
+    typedef std::vector<uint8_t> pixels_t;
+    
+    struct dictentry_t
+    {
+        pixels_t replacement; // The expanded version of this block.
+        int score; // Number of bytes that having this entry saves.
+        bool ref_encode; // Encode using references to other dictionary entries.
+        
+        dictentry_t(): score(0), ref_encode(false) {}
+    };
+    
+    struct glyphentry_t
+    {
+        pixels_t data; // The full data of the glyph.
+        std::vector<int> chars; // Characters that this glyph represents.
+        int width; // Tracking width of the character.
+    };
+    
+    struct fontinfo_t
+    {
+        std::string name; // Name of the typeface
+        int max_width; // Width of the character bounding box.
+        int max_height; // Height of the character bounding box.
+        int baseline_x; // X coordinate (from left) of the baseline.
+        int baseline_y; // Y coordinate (from top) of the baseline.
+        int line_height; // Line height (vertical advance).
+        int flags;
+    };
+    
+    static const int FLAG_MONOSPACE = 0x01;
+    static const int FLAG_BW = 0x02;
+    
+    // Construct from data in memory.
+    DataFile(const std::vector<dictentry_t> &dictionary,
+             const std::vector<glyphentry_t> &glyphs,
+             const fontinfo_t &fontinfo);
+    
+    // Save to a file (custom format)
+    void Save(std::ostream &file) const;
+
+    // Load from a file (custom format)
+    // Returns nullptr if load fails.
+    static std::unique_ptr<DataFile> Load(std::istream &file);
+    
+    // Get or set an entry in the dictionary. The size of the dictionary
+    // is constant. Entries 0 to 23 are reserved for special purposes.
+    static const size_t dictionarysize = 256 - 24;
+    const dictentry_t &GetDictionaryEntry(size_t index) const
+        { return m_dictionary.at(index); }
+    void SetDictionaryEntry(size_t index, const dictentry_t &value);
+    const std::vector<dictentry_t> &GetDictionary() const
+        { return m_dictionary; }
+        
+    // Get the index of the dictionary entry with the lowest score.
+    size_t GetLowScoreIndex() const
+        { return m_lowscoreindex; }
+    
+    // Get an entry in the glyph table.
+    size_t GetGlyphCount() const
+        { return m_glyphtable.size(); }
+    const glyphentry_t &GetGlyphEntry(size_t index) const
+        { return m_glyphtable.at(index); }
+    const std::vector<glyphentry_t> &GetGlyphTable() const
+        { return m_glyphtable; }
+    
+    // Create a map of char indices to glyph indices
+    std::map<size_t, size_t> GetCharToGlyphMap() const;
+    
+    // Get the information that applies to all glyphs.
+    const fontinfo_t &GetFontInfo() const
+        { return m_fontinfo; }
+    
+    // Show a glyph as text.
+    std::string GlyphToText(size_t index) const;
+    
+    // Get the random generator seed
+    // The seed is stored in the datafile to get deterministic behaviour
+    // for debugging.
+    uint32_t GetSeed() const { return m_seed; }
+    void SetSeed(uint32_t seed) { m_seed = seed; }
+    
+private:
+    std::vector<dictentry_t> m_dictionary;
+    std::vector<glyphentry_t> m_glyphtable;
+    fontinfo_t m_fontinfo;
+    uint32_t m_seed;
+    
+    size_t m_lowscoreindex;
+    
+    void UpdateLowScoreIndex();
+};
+
+std::ostream& operator<<(std::ostream& os, const DataFile::pixels_t& str);
+std::istream& operator>>(std::istream& is, DataFile::pixels_t& str);
+
+}
+
+#ifdef CXXTEST_RUNNING
+#include <cxxtest/TestSuite.h>
+
+using namespace mcufont;
+
+class DataFileTests: public CxxTest::TestSuite
+{
+public:
+    void testFileLoad()
+    {
+        std::istringstream s(testfile);
+        std::unique_ptr<DataFile> f = DataFile::Load(s);
+        
+        TS_ASSERT_EQUALS(f->GetFontInfo().name, "Sans Serif");
+        TS_ASSERT_EQUALS(f->GetFontInfo().max_width, 4);
+        TS_ASSERT_EQUALS(f->GetFontInfo().max_height, 6);
+        TS_ASSERT_EQUALS(f->GetDictionaryEntry(0).score, 5);
+        TS_ASSERT_EQUALS(f->GetDictionaryEntry(1).score, 13);
+        TS_ASSERT_EQUALS(f->GetGlyphCount(), 3);
+        
+        DataFile::pixels_t expected = {
+            0,15,0,15, 0,15,0,15, 0,15,0,15, 0,15,0,15, 0,15,0,15, 0,15,0,15
+        };
+        TS_ASSERT_EQUALS(f->GetGlyphEntry(0).data.size(), 24);
+        TS_ASSERT(f->GetGlyphEntry(0).data == expected);
+    }
+    
+    void testFileSave()
+    {
+        std::istringstream is1(testfile);
+        std::unique_ptr<DataFile> f1 = DataFile::Load(is1);
+        
+        std::ostringstream os;
+        f1->Save(os);
+        
+        std::string text = os.str();
+        std::istringstream is2(text);
+        std::unique_ptr<DataFile> f2 = DataFile::Load(is2);
+        
+        TS_ASSERT_EQUALS(f1->GetFontInfo().name, f2->GetFontInfo().name);
+        TS_ASSERT(f1->GetGlyphEntry(0).data == f2->GetGlyphEntry(0).data);
+    }
+    
+private:
+    static constexpr const char *testfile =
+        "Version 1\n"
+        "FontName Sans Serif\n"
+        "MaxWidth 4\n"
+        "MaxHeight 6\n"
+        "BaselineX 1\n"
+        "BaselineY 1\n"
+        "DictEntry 5 0 0F0F0\n"
+        "DictEntry 13 0 F0F0F0\n"
+        "DictEntry 1 0 F0F0F0\n"
+        "Glyph 1,2,3 4 0F0F0F0F0F0F0F0F0F0F0F0F\n"
+        "Glyph 4 4 0F0F0F0F0F0F0F0F0F0F0F0F\n"
+        "Glyph 5 4 0F0F0F0F0F0F0F0F0F0F0F0F\n";
+};
+
+#endif

tools/mcufontencoder/src/encode_rlefont.cc

@@ -0,0 +1,735 @@
+#include "encode_rlefont.hh"
+#include <algorithm>
+#include <stdexcept>
+#include "ccfixes.hh"
+
+// Number of reserved codes before the dictionary entries.
+#define DICT_START 24
+
+// Special reference to mean "fill with zeros to the end of the glyph"
+#define REF_FILLZEROS 16
+
+// RLE codes
+#define RLE_CODEMASK    0xC0
+#define RLE_VALMASK     0x3F
+#define RLE_ZEROS       0x00 // 0 to 63 zeros
+#define RLE_64ZEROS     0x40 // (1 to 64) * 64 zeros
+#define RLE_ONES        0x80 // 1 to 64 full alphas
+#define RLE_SHADE       0xC0 // 1 to 4 partial alphas
+
+// Dictionary "fill entries" for encoding bits directly.
+#define DICT_START7BIT  4
+#define DICT_START6BIT  132
+#define DICT_START5BIT  196
+#define DICT_START4BIT  228
+#define DICT_START3BIT  244
+#define DICT_START2BIT  252
+
+namespace mcufont {
+namespace rlefont {
+
+// Get bit count for the "fill entries"
+static size_t fillentry_bitcount(size_t index)
+{
+    if (index >= DICT_START2BIT)
+        return 2;
+    else if (index >= DICT_START3BIT)
+        return 3;
+    else if (index >= DICT_START4BIT)
+        return 4;
+    else if (index >= DICT_START5BIT)
+        return 5;
+    else if (index >= DICT_START6BIT)
+        return 6;
+    else
+        return 7;
+}
+
+// Count the number of equal pixels at the beginning of the pixelstring.
+static size_t prefix_length(const DataFile::pixels_t &pixels, size_t pos)
+{
+    uint8_t pixel = pixels.at(pos);
+    size_t count = 1;
+    while (pos + count < pixels.size() &&
+            pixels.at(pos + count) == pixel)
+    {
+        count++;
+    }
+    return count;
+}
+
+// Perform the RLE encoding for a dictionary entry.
+static encoded_font_t::rlestring_t encode_rle(const DataFile::pixels_t &pixels)
+{
+    encoded_font_t::rlestring_t result;
+    
+    size_t pos = 0;
+    while (pos < pixels.size())
+    {
+        uint8_t pixel = pixels.at(pos);
+        size_t count = prefix_length(pixels, pos);
+        pos += count;
+        
+        if (pixel == 0)
+        {
+            // Up to 63 zeros can be encoded with RLE_ZEROS. If there are more,
+            // encode using RLE_64ZEROS, and then whatever remains with RLE_ZEROS.
+            while (count >= 64)
+            {
+                size_t c = (count > 4096) ? 64 : (count / 64);
+                result.push_back(RLE_64ZEROS | (c - 1));
+                count -= c * 64;
+            }
+            
+            if (count)
+            {
+                result.push_back(RLE_ZEROS | count);
+            }
+        }
+        else if (pixel == 15)
+        {
+            // Encode ones.
+            while (count)
+            {
+                size_t c = (count > 64) ? 64 : count;
+                result.push_back(RLE_ONES | (c - 1));
+                count -= c;
+            }
+        }
+        else
+        {
+            // Encode shades.
+            while (count)
+            {
+                size_t c = (count > 4) ? 4 : count;
+                result.push_back(RLE_SHADE | ((c - 1) << 4) | pixel);
+                count -= c;
+            }
+        }
+    }
+    
+    return result;
+}
+
+// We use a tree structure to represent the dictionary entries.
+// Using this tree, we can perform a combined Aho-Corasick string matching
+// and breadth-first search to find the optimal encoding of glyph data.
+class DictTreeNode
+{
+public:
+    constexpr DictTreeNode():
+        m_index(-1),
+        m_ref(false),
+        m_length(0),
+        m_child0(nullptr),
+        m_child15(nullptr),
+        m_suffix(nullptr)
+        {}
+    
+    void SetChild(uint8_t p, DictTreeNode *child)
+    {
+        if (p == 0)
+            m_child0 = child;
+        else if (p == 15)
+            m_child15 = child;
+        else if (p > 15)
+            throw std::logic_error("invalid pixel alpha: " + std::to_string(p));
+        else
+        {
+            if (!m_children)
+            {
+                m_children.reset(new DictTreeNode*[14]());
+            }
+            m_children[p - 1] = child;
+        }
+    }
+    
+    DictTreeNode* GetChild(uint8_t p) const
+    { 
+        if (p == 0)
+            return m_child0;
+        else if (p == 15)
+            return m_child15;
+        else if (p > 15)
+            throw std::logic_error("invalid pixel alpha: " + std::to_string(p));
+        else if (!m_children)
+            return nullptr;
+        else
+            return m_children[p - 1];
+    }
+    
+    bool HasIntermediateChildren() const { return m_children != nullptr; }
+    
+    int GetIndex() const { return m_index; }
+    void SetIndex(int index) { m_index = index; }
+    bool GetRef() const { return m_ref; }
+    void SetRef(bool ref) { m_ref = ref; }
+    size_t GetLength() const { return m_length; }
+    void SetLength(size_t length) { m_length = length; }
+    DictTreeNode *GetSuffix() const { return m_suffix; }
+    void SetSuffix(DictTreeNode *suffix) { m_suffix = suffix; }
+    
+private:
+    // Index of dictionary entry or -1 if just a intermediate node.
+    int m_index;
+    
+    // True for ref-encoded dictionary entries. Used to avoid recursion when
+    // encoding them.
+    bool m_ref;
+    
+    // Length of the corresponding dictionary entry replacement.
+    // Equals the distance from the tree root.
+    size_t m_length;
+    
+    // Most tree nodes will only ever contains children for 0 or 15.
+    // Therefore the array for other nodes is allocated only on demand.
+    DictTreeNode *m_child0;
+    DictTreeNode *m_child15;
+    std::unique_ptr<DictTreeNode*[]> m_children;
+    
+    // Pointer to the longest suffix of this entry that exists in the
+    // dictionary.
+    DictTreeNode *m_suffix;
+};
+
+// Preallocated array for tree nodes
+class TreeAllocator
+{
+public:
+    TreeAllocator(size_t count)
+    {
+        m_storage.reset(new DictTreeNode[count]);
+        m_next = m_storage.get();
+        m_left = count;
+    }
+    
+    DictTreeNode *allocate()
+    {
+        if (m_left == 0)
+            throw std::logic_error("Ran out of preallocated entries");
+        
+        m_left--;
+        return m_next++;
+    }
+    
+private:
+    std::unique_ptr<DictTreeNode[]> m_storage;
+    DictTreeNode *m_next;
+    size_t m_left;
+};
+
+// Add a new dictionary entry to the tree. Adds the intermediate nodes, but
+// does not yet fill the suffix pointers.
+static DictTreeNode* add_tree_entry(const DataFile::pixels_t &entry, int index,
+                                    bool ref_encoded, DictTreeNode *root,
+                                    TreeAllocator &storage)
+{
+    DictTreeNode* node = root;
+    for (uint8_t p : entry)
+    {
+        DictTreeNode* branch = node->GetChild(p);
+        if (!branch)
+        {
+            branch = storage.allocate();
+            node->SetChild(p, branch);
+        }
+        
+        node = branch;
+    }
+    
+    // Replace the entry if it either does not yet have an encoding, or if
+    // the new entry is non-ref (i.e. can be used in more situations).
+    if (node->GetIndex() < 0 || (node->GetRef() && !ref_encoded))
+    {
+        node->SetIndex(index);
+        node->SetRef(ref_encoded);
+        node->SetLength(entry.size());
+    }
+    
+    return node;
+}
+
+// Walk the tree and find if the entry exists in the tree. If it does,
+// returns a pointer to it, otherwise nullptr.
+static DictTreeNode *find_tree_node(DataFile::pixels_t::const_iterator begin,
+                                    DataFile::pixels_t::const_iterator end,
+                                    DictTreeNode *root)
+{
+    DictTreeNode* node = root;
+    while (begin != end)
+    {
+        uint8_t pixel = *begin++;
+        node = node->GetChild(pixel);
+        
+        if (!node)
+            return nullptr;
+    }
+    
+    return node;
+}
+
+// Fill in the suffix pointers recursively for the given subtree.
+static void fill_tree_suffixes(DictTreeNode *root, DictTreeNode *subtree,
+    const DataFile::pixels_t &entry)
+{
+    for (size_t i = 1; i < entry.size(); i++)
+    {
+        DictTreeNode *node = find_tree_node(entry.begin() + i, entry.end(), root);
+        if (node)
+        {
+            subtree->SetSuffix(node);
+            break;
+        }
+    }
+    
+    if (!subtree->GetSuffix())
+        subtree->SetSuffix(root);
+    
+    DataFile::pixels_t newentry(entry);
+    newentry.resize(entry.size() + 1);
+    for (uint8_t i = 0; i < 16; i++)
+    {
+        // Speed-up for the common case of 0 and 15 alphas.
+        if (i == 1 && !subtree->HasIntermediateChildren())
+            i += 14;
+        
+        DictTreeNode *child = subtree->GetChild(i);
+        if (child)
+        {
+            newentry.at(entry.size()) = i;
+            fill_tree_suffixes(root, child, newentry);
+        }
+    }
+}
+
+// Construct a lookup tree from the dictionary entries.
+static DictTreeNode* construct_tree(const std::vector<DataFile::dictentry_t> &dictionary,
+                                    TreeAllocator &storage, bool fast)
+{
+    DictTreeNode* root = storage.allocate();
+    
+    // Populate the hardcoded entries for 0 to 15 alpha.
+    for (int j = 0; j < 16; j++)
+    {
+        DictTreeNode *node = storage.allocate();
+        node->SetIndex(j);
+        node->SetRef(false);
+        node->SetLength(1);
+        root->SetChild(j, node);
+    }
+    
+    // Populate the actual dictionary entries
+    size_t i = DICT_START;
+    for (DataFile::dictentry_t d : dictionary)
+    {
+        if (!d.replacement.size())
+            break;
+        
+        add_tree_entry(d.replacement, i, d.ref_encode, root, storage);
+        i++;
+    }
+    
+    if (!fast)
+    {
+        // Populate the fill entries for rest of dictionary
+        for (; i < 256; i++)
+        {
+            DataFile::pixels_t pixels;
+            size_t bitcount = fillentry_bitcount(i);
+            uint8_t byte = i - DICT_START7BIT;
+            for (size_t j = 0; j < bitcount; j++)
+            {
+                uint8_t p = (byte & (1 << j)) ? 15 : 0;
+                pixels.push_back(p);
+            }
+            
+            add_tree_entry(pixels, i, false, root, storage);
+        }
+        
+        // Fill in the suffix pointers for optimal encoding
+        DataFile::pixels_t nullentry;
+        fill_tree_suffixes(root, root, nullentry);
+    }
+    
+    return root;
+}
+
+// Structure for keeping track of the shortest encoding to reach particular
+// point of the pixel string.
+struct encoding_link_t
+{
+    // Index of the position prior to the last dictionary entry.
+    size_t previous;
+    
+    // Index of the dictionary entry that brings us to this point.
+    int index;
+    
+    // Number of links to get here from the start of the string.
+    size_t length;
+    
+    constexpr encoding_link_t(): previous(0), index(-1), length(9999999) {}
+};
+
+// Perform the reference encoding for a glyph entry (optimal version).
+// Uses a modified Aho-Corasick algorithm combined with breadth first search
+// to find the shortest representation.
+static encoded_font_t::refstring_t encode_ref_slow(const DataFile::pixels_t &pixels,
+                                                   const DictTreeNode *root,
+                                                   bool is_glyph)
+{
+    // Chain of encodings. Each entry in this array corresponds to a position
+    // in the pixel string.
+    std::unique_ptr<encoding_link_t[]> chain(new encoding_link_t[pixels.size() + 1]);
+    
+    chain[0].previous = 0;
+    chain[0].index = 0;
+    chain[0].length = 0;
+    
+    // Read the pixels one-by-one and update the encoding links accordingly.
+    const DictTreeNode *node = root;
+    for (size_t pos = 0; pos < pixels.size(); pos++)
+    {
+        uint8_t pixel = pixels.at(pos);
+        const DictTreeNode *branch = node->GetChild(pixel);
+        
+        while (!branch)
+        {
+            // Cannot expand this sequence, defer to suffix.
+            node = node->GetSuffix();
+            branch = node->GetChild(pixel);
+        }
+        
+        node = branch;
+        
+        // We have arrived at a new node, add it and any proper suffixes to
+        // the link chain.
+        const DictTreeNode *suffix = node;
+        while (suffix != root)
+        {
+            if (suffix->GetIndex() >= 0 && (is_glyph || !suffix->GetRef()))
+            {
+                encoding_link_t link;
+                link.previous = pos + 1 - suffix->GetLength();
+                link.index = suffix->GetIndex();
+                link.length = chain[link.previous].length + 1;
+                
+                if (link.length < chain[pos + 1].length)
+                    chain[pos + 1] = link;
+            }
+            suffix = suffix->GetSuffix();
+        }
+    }
+    
+    // Check if we can shorten the final encoding using REF_FILLZEROS.
+    if (is_glyph)
+    {
+        for (size_t pos = pixels.size() - 1; pos > 0; pos--)
+        {
+            if (pixels.at(pos) != 0)
+                break;
+            
+            encoding_link_t link;
+            link.previous = pos;
+            link.index = REF_FILLZEROS;
+            link.length = chain[pos].length + 1;
+            
+            if (link.length <= chain[pixels.size()].length)
+                chain[pixels.size()] = link;
+        }
+    }
+    
+    // Backtrack from the final link back to the start and construct the
+    // encoded string.
+    encoded_font_t::refstring_t result;
+    size_t len = chain[pixels.size()].length;
+    result.resize(len);
+    
+    size_t pos = pixels.size();
+    for (size_t i = len; i > 0; i--)
+    {
+        result.at(i - 1) = chain[pos].index;
+        pos = chain[pos].previous;
+    }
+    
+    return result;
+}
+
+// Walk the tree as far as possible following the given pixel string iterator.
+// Returns number of pixels encoded, and index is set to the dictionary reference.
+static size_t walk_tree(const DictTreeNode *tree,
+                        DataFile::pixels_t::const_iterator pixels,
+                        DataFile::pixels_t::const_iterator pixelsend,
+                        int &index, bool is_glyph)
+{
+    size_t best_length = 0;
+    size_t length = 0;
+    index = -1;
+    
+    const DictTreeNode* node = tree;
+    while (pixels != pixelsend)
+    {
+        uint8_t pixel = *pixels++;
+        node = node->GetChild(pixel);
+        
+        if (!node)
+            break;
+        
+        length++;
+        
+        if (is_glyph || !node->GetRef())
+        {
+            if (node->GetIndex() >= 0)
+            {
+                index = node->GetIndex();
+                best_length = length;
+            }
+        }
+    }
+    
+    if (index < 0)
+        throw std::logic_error("walk_tree failed to find a valid encoding");
+    
+    return best_length;
+}
+
+// Perform the reference encoding for a glyph entry (fast version).
+// Uses a simple greedy search to find select the encodings.
+static encoded_font_t::refstring_t encode_ref_fast(const DataFile::pixels_t &pixels,
+                                                   const DictTreeNode *tree,
+                                                   bool is_glyph)
+{
+    encoded_font_t::refstring_t result;
+    
+    // Strip any zeroes from end
+    size_t end = pixels.size();
+    
+    if (is_glyph)
+    {
+        while (end > 0 && pixels.at(end - 1) == 0) end--;
+    }
+    
+    size_t i = 0;
+    while (i < end)
+    {
+        int index;
+        i += walk_tree(tree, pixels.begin() + i, pixels.end(), index, is_glyph);
+        result.push_back(index);
+    }
+    
+    if (i < pixels.size())
+        result.push_back(REF_FILLZEROS);
+    
+    return result;
+}
+
+static encoded_font_t::refstring_t encode_ref(const DataFile::pixels_t &pixels,
+                                              const DictTreeNode *tree,
+                                              bool is_glyph, bool fast)
+{
+    if (fast)
+        return encode_ref_fast(pixels, tree, is_glyph);
+    else
+        return encode_ref_slow(pixels, tree, is_glyph);
+}
+
+// Compare dictionary entries by their coding type.
+// Sorts RLE-encoded entries first and any empty entries last.
+static bool cmp_dict_coding(const DataFile::dictentry_t &a,
+                            const DataFile::dictentry_t &b)
+{
+    if (a.replacement.size() == 0 && b.replacement.size() != 0)
+        return false;
+    else if (a.replacement.size() != 0 && b.replacement.size() == 0)
+        return true;
+    else if (a.ref_encode == false && b.ref_encode == true)
+        return true;
+    else
+        return false;
+}
+
+size_t estimate_tree_node_count(const std::vector<DataFile::dictentry_t> &dict)
+{
+    size_t count = DICT_START; // Preallocated entries
+    for (const DataFile::dictentry_t &d: dict)
+    {
+        count += d.replacement.size();
+    }
+    count += 128 * 7; // Fill entries
+    return count;
+}
+
+std::unique_ptr<encoded_font_t> encode_font(const DataFile &datafile,
+                                            bool fast)
+{
+    std::unique_ptr<encoded_font_t> result(new encoded_font_t);
+    
+    // Sort the dictionary so that RLE-coded entries come first.
+    // This way the two are easy to distinguish based on index.
+    std::vector<DataFile::dictentry_t> sorted_dict = datafile.GetDictionary();
+    std::stable_sort(sorted_dict.begin(), sorted_dict.end(), cmp_dict_coding);
+    
+    // Build the binary tree for looking up references.
+    size_t count = estimate_tree_node_count(sorted_dict);
+    TreeAllocator allocator(count);
+    DictTreeNode* tree = construct_tree(sorted_dict, allocator, fast);
+    
+    // Encode the dictionary entries, using either RLE or reference method.
+    for (const DataFile::dictentry_t &d : sorted_dict)
+    {
+        if (d.replacement.size() == 0)
+        {
+            continue;
+        }
+        else if (d.ref_encode)
+        {
+            result->ref_dictionary.push_back(encode_ref(d.replacement, tree, false, fast));
+        }
+        else
+        {
+            result->rle_dictionary.push_back(encode_rle(d.replacement));
+        }
+    }
+    
+    // Then reference-encode the glyphs
+    for (const DataFile::glyphentry_t &g : datafile.GetGlyphTable())
+    {
+        result->glyphs.push_back(encode_ref(g.data, tree, true, fast));
+    }
+    
+    // Optionally verify that the encoding was correct.
+    if (!fast)
+    {
+        for (size_t i = 0; i < datafile.GetGlyphCount(); i++)
+        {
+            std::unique_ptr<DataFile::pixels_t> decoded = 
+                decode_glyph(*result, i, datafile.GetFontInfo());
+            if (*decoded != datafile.GetGlyphEntry(i).data)
+            {
+                auto iter = std::mismatch(decoded->begin(), decoded->end(),
+                                          datafile.GetGlyphEntry(i).data.begin());
+                size_t pos = iter.first - decoded->begin();
+                throw std::logic_error("verification of glyph " + std::to_string(i) +
+                    " failed at position " + std::to_string(pos));
+            }
+        }
+    }
+    
+    return result;
+}
+
+size_t get_encoded_size(const encoded_font_t &encoded)
+{
+    size_t total = 0;
+    for (const encoded_font_t::rlestring_t &r : encoded.rle_dictionary)
+    {
+        total += r.size();
+        
+        if (r.size() != 0)
+            total += 2; // Offset table entry
+    }
+    for (const encoded_font_t::refstring_t &r : encoded.ref_dictionary)
+    {
+        total += r.size();
+        
+        if (r.size() != 0)
+            total += 2; // Offset table entry
+    }
+    for (const encoded_font_t::refstring_t &r : encoded.glyphs)
+    {
+        total += r.size();
+        total += 2; // Offset table entry
+        total += 1; // Width table entry
+    }
+    return total;
+}
+
+std::unique_ptr<DataFile::pixels_t> decode_glyph(
+    const encoded_font_t &encoded,
+    const encoded_font_t::refstring_t &refstring,
+    const DataFile::fontinfo_t &fontinfo)
+{
+    std::unique_ptr<DataFile::pixels_t> result(new DataFile::pixels_t);
+    
+    for (uint8_t ref : refstring)
+    {
+        if (ref <= 15)
+        {
+            result->push_back(ref);
+        }
+        else if (ref == REF_FILLZEROS)
+        {
+            result->resize(fontinfo.max_width * fontinfo.max_height, 0);
+        }
+        else if (ref < DICT_START)
+        {
+            throw std::logic_error("unknown code: " + std::to_string(ref));
+        }
+        else if (ref - DICT_START < (int)encoded.rle_dictionary.size())
+        {
+            for (uint8_t rle : encoded.rle_dictionary.at(ref - DICT_START))
+            {
+                if ((rle & RLE_CODEMASK) == RLE_ZEROS)
+                {
+                    for (int i = 0; i < (rle & RLE_VALMASK); i++)
+                    {
+                        result->push_back(0);
+                    }
+                }
+                else if ((rle & RLE_CODEMASK) == RLE_64ZEROS)
+                {
+                    for (int i = 0; i < ((rle & RLE_VALMASK) + 1) * 64; i++)
+                    {
+                        result->push_back(0);
+                    }
+                }
+                else if ((rle & RLE_CODEMASK) == RLE_ONES)
+                {
+                    for (int i = 0; i < (rle & RLE_VALMASK) + 1; i++)
+                    {
+                        result->push_back(15);
+                    }
+                }
+                else if ((rle & RLE_CODEMASK) == RLE_SHADE)
+                {
+                    uint8_t count, alpha;
+                    count = ((rle & RLE_VALMASK) >> 4) + 1;
+                    alpha = ((rle & RLE_VALMASK) & 0xF);
+                    for (int i = 0; i < count; i++)
+                    {
+                        result->push_back(alpha);
+                    }
+                }
+            }
+        }
+        else if (ref - DICT_START - encoded.rle_dictionary.size() < encoded.ref_dictionary.size())
+        {
+            size_t index = ref - DICT_START - encoded.rle_dictionary.size();
+            std::unique_ptr<DataFile::pixels_t> part =
+                decode_glyph(encoded, encoded.ref_dictionary.at(index),
+                             fontinfo);
+            result->insert(result->end(), part->begin(), part->end());
+        }
+        else
+        {
+            size_t bitcount = fillentry_bitcount(ref);
+            
+            uint8_t byte = ref - DICT_START7BIT;
+            for (size_t i = 0; i < bitcount; i++)
+            {
+                uint8_t p = (byte & (1 << i)) ? 15 : 0;
+                result->push_back(p);
+            }
+        }
+    }
+    
+    return result;
+}
+
+std::unique_ptr<DataFile::pixels_t> decode_glyph(
+    const encoded_font_t &encoded, size_t index,
+    const DataFile::fontinfo_t &fontinfo)
+{
+    return decode_glyph(encoded, encoded.glyphs.at(index), fontinfo);
+}
+
+}}

tools/mcufontencoder/src/encode_rlefont.hh

@@ -0,0 +1,126 @@
+// Given a dictionary and glyphs, encode the data for all the glyphs.
+
+#pragma once
+
+#include "datafile.hh"
+#include <vector>
+#include <memory>
+
+namespace mcufont {
+namespace rlefont {
+
+struct encoded_font_t
+{
+    // RLE-encoded format for storing the dictionary entries.
+    // Each item is a byte. Top bit means the value in the original bitstream,
+    // and the bottom 7 bits store the repetition count.
+    typedef std::vector<uint8_t> rlestring_t;
+    
+    // Reference encoded format for storing the glyphs.
+    // Each item is a reference to the dictionary.
+    // Values 0 and 1 are hardcoded to mean 0 and 1.
+    // All other values mean dictionary entry at (i-2).
+    typedef std::vector<uint8_t> refstring_t;
+    
+    std::vector<rlestring_t> rle_dictionary;
+    std::vector<refstring_t> ref_dictionary;
+    std::vector<refstring_t> glyphs;
+};
+
+// Encode all the glyphs.
+std::unique_ptr<encoded_font_t> encode_font(const DataFile &datafile,
+                                            bool fast = true);
+
+// Sum up the total size of the encoded glyphs + dictionary.
+size_t get_encoded_size(const encoded_font_t &encoded);
+
+inline size_t get_encoded_size(const DataFile &datafile, bool fast = true)
+{
+    std::unique_ptr<encoded_font_t> e = encode_font(datafile, fast);
+    return get_encoded_size(*e);
+}
+
+// Decode a single glyph (for verification).
+std::unique_ptr<DataFile::pixels_t> decode_glyph(
+    const encoded_font_t &encoded,
+    const encoded_font_t::refstring_t &refstring,
+    const DataFile::fontinfo_t &fontinfo);
+
+// Decode a single glyph (for verification).
+std::unique_ptr<DataFile::pixels_t> decode_glyph(
+    const encoded_font_t &encoded, size_t index,
+    const DataFile::fontinfo_t &fontinfo);
+
+}}
+
+
+#ifdef CXXTEST_RUNNING
+#include <cxxtest/TestSuite.h>
+
+using namespace mcufont;
+using namespace mcufont::rlefont;
+
+class RLEFontEncodeTests: public CxxTest::TestSuite
+{
+public:
+    void testEncode()
+    {
+        std::istringstream s(testfile);
+        std::unique_ptr<DataFile> f = DataFile::Load(s);
+        std::unique_ptr<encoded_font_t> e = encode_font(*f, false);
+        
+        TS_ASSERT_EQUALS(e->glyphs.size(), 3);
+        
+        // Expected values for dictionary
+        encoded_font_t::rlestring_t dict0 = {0x01, 0xCE, 0x01, 0xCE};
+        encoded_font_t::rlestring_t dict1 = {0x0C};
+        encoded_font_t::rlestring_t dict2 = {0xFE};
+        encoded_font_t::refstring_t dict3 = {24, 24};
+        
+        TS_ASSERT(e->rle_dictionary.at(0) == dict0);
+        TS_ASSERT(e->rle_dictionary.at(1) == dict1);
+        TS_ASSERT(e->rle_dictionary.at(2) == dict2);
+        TS_ASSERT(e->ref_dictionary.at(0) == dict3);
+        
+        // Expected values for glyphs
+        encoded_font_t::refstring_t glyph0 = {27, 27, 27};
+        encoded_font_t::refstring_t glyph1 = {24, 0, 132, 25, 14};
+        encoded_font_t::refstring_t glyph2 = {228, 26, 244, 14, 14, 14, 228, 26, 16};
+        
+        TS_ASSERT_EQUALS(e->glyphs.at(0), glyph0);
+        TS_ASSERT_EQUALS(e->glyphs.at(1), glyph1);
+        TS_ASSERT_EQUALS(e->glyphs.at(2), glyph2);
+    }
+    
+    void testDecode()
+    {
+        std::istringstream s(testfile);
+        std::unique_ptr<DataFile> f = DataFile::Load(s);
+        std::unique_ptr<encoded_font_t> e = encode_font(*f, false);
+        
+        for (size_t i = 0; i < 3; i++)
+        {
+            std::unique_ptr<DataFile::pixels_t> dec;
+            dec = decode_glyph(*e, i, f->GetFontInfo());
+            
+            TS_ASSERT_EQUALS(*dec, f->GetGlyphEntry(i).data);
+        }
+    }
+    
+private:
+    static constexpr const char *testfile =
+        "Version 1\n"
+        "FontName Sans Serif\n"
+        "MaxWidth 4\n"
+        "MaxHeight 6\n"
+        "BaselineX 1\n"
+        "BaselineY 1\n"
+        "DictEntry 1 0 0E0E\n"
+        "DictEntry 1 0 000000000000\n"
+        "DictEntry 1 0 EEEE\n"
+        "DictEntry 1 1 0E0E0E0E\n"
+        "Glyph 0 4 0E0E0E0E0E0E0E0E0E0E0E0E\n"
+        "Glyph 1 4 0E0E0000000000000000000E\n"
+        "Glyph 2 4 0000EEEE000EEE0000EEEE00\n";
+};
+#endif

tools/mcufontencoder/src/export_bwfont.cc

@@ -0,0 +1,247 @@
+#include "export_bwfont.hh"
+#include <vector>
+#include <iomanip>
+#include <map>
+#include <set>
+#include <algorithm>
+#include <string>
+#include <cctype>
+#include "exporttools.hh"
+#include "importtools.hh"
+#include "ccfixes.hh"
+
+#define BWFONT_FORMAT_VERSION 4
+
+namespace mcufont {
+namespace bwfont {
+
+static void encode_glyph(const DataFile::glyphentry_t &glyph,
+                         const DataFile::fontinfo_t &fontinfo,
+                         std::vector<unsigned> &dest,
+                         int num_cols)
+{
+    const int threshold = 8;
+    
+    // Find the number of columns in the glyph data
+    if (num_cols == 0)
+    {
+        for (int x = 0; x < fontinfo.max_width; x++)
+        {
+            for (int y = 0; y < fontinfo.max_height; y++)
+            {
+                size_t index = y * fontinfo.max_width + x;
+                if (glyph.data.at(index) >= threshold)
+                    num_cols = x + 1;
+            }
+        }
+    }
+    
+    // Write the bits that compose the glyph
+    for (int x = 0; x < num_cols; x++)
+    {
+        for (int y = 0; y < fontinfo.max_height; y+= 8)
+        {
+            size_t remain = std::min(8, fontinfo.max_height - y);
+            uint8_t byte = 0;
+            for (size_t i = 0; i < remain; i++)
+            {
+                size_t index = (y + i) * fontinfo.max_width + x;
+                if (glyph.data.at(index) >= threshold)
+                {
+                    byte |= (1 << i);
+                }
+            }
+            dest.push_back(byte);
+        }
+    }
+}
+
+struct cropinfo_t
+{
+    size_t offset_x;
+    size_t offset_y;
+    size_t height_bytes;
+    size_t height_pixels;
+    size_t width;
+};
+
+static void encode_character_range(std::ostream &out,
+                                   const std::string &name,
+                                   const DataFile &datafile,
+                                   const char_range_t &range,
+                                   unsigned range_index,
+                                   cropinfo_t &cropinfo)
+{
+    std::vector<DataFile::glyphentry_t> glyphs;
+    bool constant_width = true;
+    int width = datafile.GetGlyphEntry(range.glyph_indices[0]).width;
+    
+    // Copy all the glyphs in this range for the purpose of cropping them.
+    for (int glyph_index: range.glyph_indices)
+    {
+        if (glyph_index < 0)
+        {
+            // Missing glyph
+            DataFile::glyphentry_t dummy = {};
+            glyphs.push_back(dummy);
+        }
+        else
+        {
+            auto glyph = datafile.GetGlyphEntry(glyph_index);
+            glyphs.push_back(glyph);
+            
+            if (glyph.width != width)
+            {
+                constant_width = false;
+                width = 0;
+            }
+        }
+    }
+    
+    // Crop the glyphs in this range. Getting rid of a few rows at top
+    // or left can save a bunch of bytes with minimal cost.
+    DataFile::fontinfo_t old_fi = datafile.GetFontInfo();
+    DataFile::fontinfo_t new_fi = old_fi;
+    crop_glyphs(glyphs, new_fi);
+    
+    if (new_fi.max_width != width)
+    {
+        constant_width = false;
+        width = 0;
+    }
+    
+    // Fill in the crop information
+    cropinfo.offset_x = old_fi.baseline_x - new_fi.baseline_x;
+    cropinfo.offset_y = old_fi.baseline_y - new_fi.baseline_y;
+    cropinfo.height_pixels = new_fi.max_height;
+    cropinfo.height_bytes = (cropinfo.height_pixels + 7) / 8;
+    cropinfo.width = width;
+    
+    // Then format and write out the glyph data
+    std::vector<unsigned> offsets;
+    std::vector<unsigned> data;
+    std::vector<unsigned> widths;
+    size_t stride = cropinfo.height_bytes;
+    
+    for (const DataFile::glyphentry_t &g : glyphs)
+    {
+        offsets.push_back(data.size() / stride);
+        widths.push_back(g.width);
+        encode_glyph(g, new_fi, data, width);
+    }    
+    offsets.push_back(data.size() / stride);
+    
+    write_const_table(out, data, "uint8_t", "mf_bwfont_" + name + "_glyph_data_" + std::to_string(range_index));
+    
+    if (!constant_width)
+    {
+        write_const_table(out, offsets, "uint16_t", "mf_bwfont_" + name + "_glyph_offsets_" + std::to_string(range_index), 4);
+        write_const_table(out, widths, "uint8_t", "mf_bwfont_" + name + "_glyph_widths_" + std::to_string(range_index));
+    }
+}
+    
+void write_source(std::ostream &out, std::string name, const DataFile &datafile)
+{
+    name = filename_to_identifier(name);
+    
+    out << std::endl;
+    out << std::endl;
+    out << "/* Start of automatically generated font definition for " << name << ". */" << std::endl;
+    out << std::endl;
+    
+    out << "#ifndef MF_BWFONT_INTERNALS" << std::endl;
+    out << "#define MF_BWFONT_INTERNALS" << std::endl;
+    out << "#endif" << std::endl;
+    out << "#include \"mf_bwfont.h\"" << std::endl;
+    out << std::endl;
+    
+    out << "#ifndef MF_BWFONT_VERSION_" << BWFONT_FORMAT_VERSION << "_SUPPORTED" << std::endl;
+    out << "#error The font file is not compatible with this version of mcufont." << std::endl;
+    out << "#endif" << std::endl;
+    out << std::endl;
+    
+    // Split the characters into ranges
+    DataFile::fontinfo_t f = datafile.GetFontInfo();
+    size_t glyph_size = f.max_width * ((f.max_height + 7) / 8);
+    auto get_glyph_size = [=](size_t i) { return glyph_size; };
+    std::vector<char_range_t> ranges = compute_char_ranges(datafile,
+        get_glyph_size, 65536, 16);
+
+    // Write out glyph data for character ranges
+    std::vector<cropinfo_t> crops;
+    for (size_t i = 0; i < ranges.size(); i++)
+    {
+        cropinfo_t cropinfo;
+        encode_character_range(out, name, datafile, ranges.at(i), i, cropinfo);
+        crops.push_back(cropinfo);
+    }
+    
+    // Write out a table describing the character ranges
+    out << "static const struct mf_bwfont_char_range_s mf_bwfont_" + name + "_char_ranges[] = {" << std::endl;
+    for (size_t i = 0; i < ranges.size(); i++)
+    {
+        std::string offsets = (crops[i].width) ? "0" : "mf_bwfont_" + name + "_glyph_offsets_" + std::to_string(i);
+        std::string widths = (crops[i].width) ? "0" : "mf_bwfont_" + name + "_glyph_widths_" + std::to_string(i);
+        
+        out << "    {" << std::endl;
+        out << "        " << ranges.at(i).first_char << ", /* first char */" << std::endl;
+        out << "        " << ranges.at(i).char_count << ", /* char count */" << std::endl;
+        out << "        " << crops[i].offset_x << ", /* offset x */" << std::endl;
+        out << "        " << crops[i].offset_y << ", /* offset y */" << std::endl;
+        out << "        " << crops[i].height_bytes << ", /* height in bytes */" << std::endl;
+        out << "        " << crops[i].height_pixels << ", /* height in pixels */" << std::endl;
+        out << "        " << crops[i].width << ", /* width */" << std::endl;
+        out << "        " << widths << ", /* glyph widths */" << std::endl;
+        out << "        " << offsets << ", /* glyph offsets */" << std::endl;
+        out << "        " << "mf_bwfont_" << name << "_glyph_data_" << i << ", /* glyph data */" << std::endl;
+        out << "    }," << std::endl;
+    }
+    out << "};" << std::endl;
+    out << std::endl;
+    
+    // Fonts in this format are always black & white
+    int flags = datafile.GetFontInfo().flags | DataFile::FLAG_BW;
+    
+    // Pull it all together in the rlefont_s structure.
+    out << "const struct mf_bwfont_s mf_bwfont_" << name << " = {" << std::endl;
+    out << "    {" << std::endl;
+    out << "    " << "\"" << datafile.GetFontInfo().name << "\"," << std::endl;
+    out << "    " << "\"" << name << "\"," << std::endl;
+    out << "    " << datafile.GetFontInfo().max_width << ", /* width */" << std::endl;
+    out << "    " << datafile.GetFontInfo().max_height << ", /* height */" << std::endl;
+    out << "    " << get_min_x_advance(datafile) << ", /* min x advance */" << std::endl;
+    out << "    " << get_max_x_advance(datafile) << ", /* max x advance */" << std::endl;
+    out << "    " << datafile.GetFontInfo().baseline_x << ", /* baseline x */" << std::endl;
+    out << "    " << datafile.GetFontInfo().baseline_y << ", /* baseline y */" << std::endl;
+    out << "    " << datafile.GetFontInfo().line_height << ", /* line height */" << std::endl;
+    out << "    " << flags << ", /* flags */" << std::endl;
+    out << "    " << select_fallback_char(datafile) << ", /* fallback character */" << std::endl;
+    out << "    " << "&mf_bwfont_character_width," << std::endl;
+    out << "    " << "&mf_bwfont_render_character," << std::endl;
+    out << "    }," << std::endl;
+    
+    out << "    " << BWFONT_FORMAT_VERSION << ", /* version */" << std::endl;
+    out << "    " << ranges.size() << ", /* char range count */" << std::endl;
+    out << "    " << "mf_bwfont_" << name << "_char_ranges," << std::endl;
+    out << "};" << std::endl;
+    
+    // Write the font lookup structure
+    out << std::endl;
+    out << "#ifdef MF_INCLUDED_FONTS" << std::endl;
+    out << "/* List entry for searching fonts by name. */" << std::endl;
+    out << "static const struct mf_font_list_s mf_bwfont_" << name << "_listentry = {" << std::endl;
+    out << "    MF_INCLUDED_FONTS," << std::endl;
+    out << "    (struct mf_font_s*)&mf_bwfont_" << name << std::endl;
+    out << "};" << std::endl;
+    out << "#undef MF_INCLUDED_FONTS" << std::endl;
+    out << "#define MF_INCLUDED_FONTS (&mf_bwfont_" << name << "_listentry)" << std::endl;
+    out << "#endif" << std::endl;
+    
+    out << std::endl;
+    out << std::endl;
+    out << "/* End of automatically generated font definition for " << name << ". */" << std::endl;
+    out << std::endl;
+}
+    
+    
+}}

tools/mcufontencoder/src/export_bwfont.hh

@@ -0,0 +1,16 @@
+// Write out the encoded data in C source code files for mf_bwfont format.
+
+#pragma once
+
+#include "datafile.hh"
+#include <iostream>
+
+namespace mcufont {
+namespace bwfont {
+
+void write_header(std::ostream &out, std::string name, const DataFile &datafile);
+
+void write_source(std::ostream &out, std::string name, const DataFile &datafile);
+
+} }
+

tools/mcufontencoder/src/export_rlefont.cc

@@ -0,0 +1,181 @@
+#include "export_rlefont.hh"
+#include <vector>
+#include <iomanip>
+#include <map>
+#include <set>
+#include <algorithm>
+#include <string>
+#include <cctype>
+#include "exporttools.hh"
+#include "ccfixes.hh"
+
+#define RLEFONT_FORMAT_VERSION 4
+
+namespace mcufont {
+namespace rlefont {
+
+// Encode the dictionary entries and the offsets to them.
+// Generates tables dictionary_data and dictionary_offsets.
+static void encode_dictionary(std::ostream &out,
+                              const std::string &name,
+                              const DataFile &datafile,
+                              const encoded_font_t &encoded)
+{
+    std::vector<unsigned> offsets;
+    std::vector<unsigned> data;
+    for (const encoded_font_t::rlestring_t &r : encoded.rle_dictionary)
+    {
+        offsets.push_back(data.size());
+        data.insert(data.end(), r.begin(), r.end());
+    }
+    
+    for (const encoded_font_t::refstring_t &r : encoded.ref_dictionary)
+    {
+        offsets.push_back(data.size());
+        data.insert(data.end(), r.begin(), r.end());
+    }
+    offsets.push_back(data.size());
+    
+    write_const_table(out, data, "uint8_t", "mf_rlefont_" + name + "_dictionary_data");
+    write_const_table(out, offsets, "uint16_t", "mf_rlefont_" + name + "_dictionary_offsets", 4);
+}
+
+// Encode the data tables for a single character range.
+// Generates tables glyph_data_i and glyph_offsets_i.
+static void encode_character_range(std::ostream &out,
+                              const std::string &name,
+                              const DataFile &datafile,
+                              const encoded_font_t& encoded,
+                              const char_range_t& range,
+                              unsigned range_index)
+{
+    std::vector<unsigned> offsets;
+    std::vector<unsigned> data;
+    std::map<size_t, unsigned> already_encoded;
+    
+    for (int glyph_index : range.glyph_indices)
+    {
+        if (already_encoded.count(glyph_index))
+        {
+            offsets.push_back(already_encoded[glyph_index]);
+        }
+        else
+        {
+            encoded_font_t::refstring_t r;
+            int width = 0;
+            
+            if (glyph_index >= 0)
+            {
+                r = encoded.glyphs[glyph_index];
+                width = datafile.GetGlyphEntry(glyph_index).width;
+            }
+            
+            offsets.push_back(data.size());
+            already_encoded[glyph_index] = data.size();
+            
+            data.push_back(width);
+            data.insert(data.end(), r.begin(), r.end());
+        }
+    }
+    
+    write_const_table(out, data, "uint8_t", "mf_rlefont_" + name + "_glyph_data_" + std::to_string(range_index));
+    write_const_table(out, offsets, "uint16_t", "mf_rlefont_" + name + "_glyph_offsets_" + std::to_string(range_index), 4);
+}
+
+void write_source(std::ostream &out, std::string name, const DataFile &datafile)
+{
+    name = filename_to_identifier(name);
+    std::unique_ptr<encoded_font_t> encoded = encode_font(datafile, false);
+    
+    out << std::endl;
+    out << std::endl;
+    out << "/* Start of automatically generated font definition for " << name << ". */" << std::endl;
+    out << std::endl;
+    
+    out << "#ifndef MF_RLEFONT_INTERNALS" << std::endl;
+    out << "#define MF_RLEFONT_INTERNALS" << std::endl;
+    out << "#endif" << std::endl;
+    out << "#include \"mf_rlefont.h\"" << std::endl;
+    out << std::endl;
+    
+    out << "#ifndef MF_RLEFONT_VERSION_" << RLEFONT_FORMAT_VERSION << "_SUPPORTED" << std::endl;
+    out << "#error The font file is not compatible with this version of mcufont." << std::endl;
+    out << "#endif" << std::endl;
+    out << std::endl;
+    
+    // Write out the dictionary entries
+    encode_dictionary(out, name, datafile, *encoded);
+    
+    // Split the characters into ranges
+    auto get_glyph_size = [&encoded](size_t i)
+    {
+        return encoded->glyphs[i].size();
+    };
+    std::vector<char_range_t> ranges = compute_char_ranges(datafile,
+        get_glyph_size, 65536, 16);
+
+    // Write out glyph data for character ranges
+    for (size_t i = 0; i < ranges.size(); i++)
+    {
+        encode_character_range(out, name, datafile, *encoded, ranges.at(i), i);
+    }
+    
+    // Write out a table describing the character ranges
+    out << "static const struct mf_rlefont_char_range_s mf_rlefont_" << name << "_char_ranges[] = {" << std::endl;
+    for (size_t i = 0; i < ranges.size(); i++)
+    {
+        out << "    {" << ranges.at(i).first_char
+            << ", " << ranges.at(i).char_count
+            << ", mf_rlefont_" << name << "_glyph_offsets_" << i
+            << ", mf_rlefont_" << name << "_glyph_data_" << i << "}," << std::endl; 
+    }
+    out << "};" << std::endl;
+    out << std::endl;
+    
+    // Pull it all together in the rlefont_s structure.
+    out << "const struct mf_rlefont_s mf_rlefont_" << name << " = {" << std::endl;
+    out << "    {" << std::endl;
+    out << "    " << "\"" << datafile.GetFontInfo().name << "\"," << std::endl;
+    out << "    " << "\"" << name << "\"," << std::endl;
+    out << "    " << datafile.GetFontInfo().max_width << ", /* width */" << std::endl;
+    out << "    " << datafile.GetFontInfo().max_height << ", /* height */" << std::endl;
+    out << "    " << get_min_x_advance(datafile) << ", /* min x advance */" << std::endl;
+    out << "    " << get_max_x_advance(datafile) << ", /* max x advance */" << std::endl;
+    out << "    " << datafile.GetFontInfo().baseline_x << ", /* baseline x */" << std::endl;
+    out << "    " << datafile.GetFontInfo().baseline_y << ", /* baseline y */" << std::endl;
+    out << "    " << datafile.GetFontInfo().line_height << ", /* line height */" << std::endl;
+    out << "    " << datafile.GetFontInfo().flags << ", /* flags */" << std::endl;
+    out << "    " << select_fallback_char(datafile) << ", /* fallback character */" << std::endl;
+    out << "    " << "&mf_rlefont_character_width," << std::endl;
+    out << "    " << "&mf_rlefont_render_character," << std::endl;
+    out << "    }," << std::endl;
+    
+    out << "    " << RLEFONT_FORMAT_VERSION << ", /* version */" << std::endl;
+    out << "    " << "mf_rlefont_" << name << "_dictionary_data," << std::endl;
+    out << "    " << "mf_rlefont_" << name << "_dictionary_offsets," << std::endl;
+    out << "    " << encoded->rle_dictionary.size() << ", /* rle dict count */" << std::endl;
+    out << "    " << encoded->ref_dictionary.size() + encoded->rle_dictionary.size() << ", /* total dict count */" << std::endl;
+    out << "    " << ranges.size() << ", /* char range count */" << std::endl;
+    out << "    " << "mf_rlefont_" << name << "_char_ranges," << std::endl;
+    out << "};" << std::endl;
+    
+    // Write the font lookup structure
+    out << std::endl;
+    out << "#ifdef MF_INCLUDED_FONTS" << std::endl;
+    out << "/* List entry for searching fonts by name. */" << std::endl;
+    out << "static const struct mf_font_list_s mf_rlefont_" << name << "_listentry = {" << std::endl;
+    out << "    MF_INCLUDED_FONTS," << std::endl;
+    out << "    (struct mf_font_s*)&mf_rlefont_" << name << std::endl;
+    out << "};" << std::endl;
+    out << "#undef MF_INCLUDED_FONTS" << std::endl;
+    out << "#define MF_INCLUDED_FONTS (&mf_rlefont_" << name << "_listentry)" << std::endl;
+    out << "#endif" << std::endl;
+    
+    out << std::endl;
+    out << std::endl;
+    out << "/* End of automatically generated font definition for " << name << ". */" << std::endl;
+    out << std::endl;
+}
+
+}}
+

tools/mcufontencoder/src/export_rlefont.hh

@@ -0,0 +1,15 @@
+// Write out the encoded data in C source code files for the mf_rlefont format.
+
+#pragma once
+
+#include "datafile.hh"
+#include "encode_rlefont.hh"
+#include <iostream>
+
+namespace mcufont {
+namespace rlefont {
+
+void write_source(std::ostream &out, std::string name, const DataFile &datafile);
+
+} }
+

tools/mcufontencoder/src/exporttools.cc

@@ -0,0 +1,179 @@
+#include "exporttools.hh"
+#include <iomanip>
+#include <set>
+
+namespace mcufont {
+    
+    
+// Convert a file name to a valid C identifier
+std::string filename_to_identifier(std::string name)
+{
+    // If the name contains path separators (/ or \), take only the last part.
+    size_t pos = name.find_last_of("/\\");
+    if (pos != std::string::npos)
+        name = name.substr(pos + 1);
+    
+    // If the name contains a file extension, strip it.
+    pos = name.find_first_of(".");
+    if (pos != std::string::npos)
+        name = name.substr(0, pos);
+    
+    // Replace any special characters with _.
+    for (pos = 0; pos < name.size(); pos++)
+    {
+        if (!isalnum(name.at(pos)))
+            name.at(pos) = '_';
+    }
+    
+    return name;
+}
+
+// Write a vector of integers as line-wrapped hex/integer data for initializing const array.
+void wordwrap_vector(std::ostream &out, const std::vector<unsigned> &data,
+                     const std::string &prefix, size_t width)
+{
+    int values_per_column = (width <= 2) ? 16 : 8;
+    
+    std::ios::fmtflags flags(out.flags());
+    out << prefix;
+    out << std::hex << std::setfill('0');
+    for (size_t i = 0; i < data.size(); i++)
+    {
+        if (i % values_per_column == 0 && i != 0)
+            out << std::endl << prefix;
+        
+        out << "0x" << std::setw(width) << (int)data.at(i) << ", ";
+    }
+    out.flags(flags);
+}
+
+// Write a vector of integers as a C constant array of given datatype.
+ void write_const_table(std::ostream &out, const std::vector<unsigned> &data,
+                        const std::string &datatype, const std::string &tablename,
+                        size_t width)
+{
+    out << "static const " << datatype << " " << tablename;
+    out << "[" << data.size() << "] = {" << std::endl;
+    wordwrap_vector(out, data, "    ", width);
+    out << std::endl << "};" << std::endl;
+    out << std::endl;
+}    
+
+int get_min_x_advance(const DataFile &datafile)
+{
+    int min = datafile.GetGlyphEntry(0).width;
+    
+    for (const DataFile::glyphentry_t &g : datafile.GetGlyphTable())
+    {
+        if (min > g.width)
+            min = g.width;
+    }
+    
+    return min;
+}
+
+int get_max_x_advance(const DataFile &datafile)
+{
+    int max = 0;
+    
+    for (const DataFile::glyphentry_t &g : datafile.GetGlyphTable())
+    {
+        if (max < g.width)
+            max = g.width;
+    }
+    
+    return max;
+}
+
+// Select the character to use as a fallback.
+int select_fallback_char(const DataFile &datafile)
+{
+    std::set<int> chars;
+    
+    size_t i = 0;
+    for (const DataFile::glyphentry_t &g: datafile.GetGlyphTable())
+    {
+        for (size_t c: g.chars)
+        {
+            chars.insert(c);
+        }
+        i++;
+    }
+    
+    if (chars.count(0xFFFD))
+        return 0xFFFD; // Unicode replacement character
+    
+    if (chars.count(0))
+        return 0; // Used by many BDF fonts as replacement char
+    
+    if (chars.count('?'))
+        return '?';
+    
+    return ' ';
+}
+
+// Decide how to best divide the characters in the font into ranges.
+// Limitations are:
+//  - Gaps longer than minimum_gap should result in separate ranges.
+//  - Each range can have encoded data size of at most maximum_size.
+std::vector<char_range_t> compute_char_ranges(const DataFile &datafile,
+    std::function<size_t(size_t)> get_encoded_glyph_size,
+    size_t maximum_size,
+    size_t minimum_gap)
+{
+    std::vector<char_range_t> result;
+    std::map<size_t, size_t> char_to_glyph = datafile.GetCharToGlyphMap();
+    std::vector<size_t> chars;
+    
+    // Get list of all characters in numeric order.
+    for (auto iter : char_to_glyph)
+        chars.push_back(iter.first);
+    
+    // Pick out ranges until we have processed all characters
+    size_t i = 0;
+    while (i < chars.size())
+    {
+        char_range_t range;
+        range.first_char = chars.at(i);
+        
+        // Find the point where there is a gap larger than minimum_gap.
+        i++;
+        while (i < chars.size() && chars.at(i) - chars.at(i - 1) < minimum_gap)
+            i++;
+        
+        uint16_t last_char = chars.at(i - 1);
+        
+        // Then store the indices of glyphs for each character
+        size_t data_length = 0;
+        for (size_t j = range.first_char; j <= last_char; j++)
+        {
+            if (char_to_glyph.count(j) == 0)
+            {
+                // Missing character
+                range.glyph_indices.push_back(-1);
+                continue;
+            }
+            
+            int glyph_index = char_to_glyph[j];
+            
+            // Monitor the amount of the data in the range and split it
+            // if it grows too large.
+            data_length += get_encoded_glyph_size(glyph_index);
+            if (data_length > maximum_size)
+            {
+                last_char = j - 1;
+                break;
+            }
+            
+            range.glyph_indices.push_back(glyph_index);
+        }
+        
+        range.char_count = last_char - range.first_char + 1;
+        result.push_back(range);
+    }
+    
+    return result;
+}
+    
+    
+}

tools/mcufontencoder/src/exporttools.hh

@@ -0,0 +1,52 @@
+// Utility functions for exporting to C source code files.
+
+#pragma once
+#include <string>
+#include <vector>
+#include <iostream>
+#include <functional>
+#include "datafile.hh"
+
+namespace mcufont {
+    
+// Convert a file name to a valid C identifier
+std::string filename_to_identifier(std::string name);
+
+// Write a vector of integers as line-wrapped hex/integer data for initializing const array.
+void wordwrap_vector(std::ostream &out, const std::vector<unsigned> &data,
+                     const std::string &prefix, size_t width = 2);
+
+// Write a vector of integers as a C constant array of given datatype.
+void write_const_table(std::ostream &out, const std::vector<unsigned> &data,
+                       const std::string &datatype, const std::string &tablename,
+                       size_t width = 2);
+
+// Get minimum tracking width of font
+int get_min_x_advance(const DataFile &datafile);
+
+// Get maximum tracking width of font
+int get_max_x_advance(const DataFile &datafile);
+
+// Select the character to use as a fallback.
+int select_fallback_char(const DataFile &datafile);
+
+// Structure to represent one consecutive range of characters.
+struct char_range_t
+{
+    uint16_t first_char;
+    uint16_t char_count;
+    std::vector<int> glyph_indices;
+    
+    char_range_t(): first_char(0), char_count(0) {}
+};
+
+// Decide how to best divide the characters in the font into ranges.
+// Limitations are:
+//  - Gaps longer than minimum_gap should result in separate ranges.
+//  - Each range can have encoded data size of at most maximum_size.
+std::vector<char_range_t> compute_char_ranges(const DataFile &datafile,
+    std::function<size_t(size_t)> get_encoded_glyph_size,
+    size_t maximum_size,
+    size_t minimum_gap);
+
+}

tools/mcufontencoder/src/freetype_import.cc

@@ -0,0 +1,177 @@
+#include "freetype_import.hh"
+#include "importtools.hh"
+#include <map>
+#include <string>
+#include <stdexcept>
+#include <iostream>
+#include "ccfixes.hh"
+
+#include <ft2build.h>
+#include FT_FREETYPE_H
+
+#undef __FTERRORS_H__
+#define FT_ERRORDEF( e, v, s )  std::make_pair( e, s ),
+#define FT_ERROR_START_LIST static const std::map<FT_Error, std::string> ft_errors {
+#define FT_ERROR_END_LIST };    
+#include FT_ERRORS_H
+
+namespace mcufont {
+
+static void checkFT(FT_Error error)
+{
+    if (error != 0)
+    {
+        if (ft_errors.count(error))
+            throw std::runtime_error("libfreetype error " +
+                std::to_string(error) + ": " + ft_errors.at(error));
+        else
+            throw std::runtime_error("unknown libfreetype error " +
+                std::to_string(error));
+    }
+}
+
+// Automatically allocated & freed wrapper for FT_Library
+class _FT_Library
+{
+public:
+    _FT_Library() { checkFT(FT_Init_FreeType(&m_lib)); }
+    ~_FT_Library() { checkFT(FT_Done_FreeType(m_lib)); }
+    operator FT_Library() { return m_lib; }
+
+private:
+    FT_Library m_lib;
+};
+
+// Automatically allocated & freed wrapper for FT_Face
+class _FT_Face
+{
+public:
+    _FT_Face(FT_Library lib, const std::vector<char> &data)
+    {
+        checkFT(FT_New_Memory_Face(lib, (const unsigned char *)&data[0],
+                                   data.size(), 0, &m_face));
+    }
+    ~_FT_Face() { checkFT(FT_Done_Face(m_face)); }
+    operator FT_Face() { return m_face; }
+    FT_Face operator->() { return m_face; }
+
+private:
+    FT_Face m_face;
+};
+
+// Read all the data from a file into a memory buffer.
+static void readfile(std::istream &file, std::vector<char> &data)
+{
+    while (file.good())
+    {
+        const size_t blocksize = 4096;
+        size_t oldsize = data.size();
+        data.resize(oldsize + blocksize);
+        file.read(&data[oldsize], blocksize);
+        data.resize(oldsize + file.gcount());
+    }
+}
+
+std::unique_ptr<DataFile> LoadFreetype(std::istream &file, int size, bool bw)
+{
+    std::vector<char> data;
+    readfile(file, data);
+    
+    _FT_Library lib;
+    _FT_Face face(lib, data);
+    
+    checkFT(FT_Set_Pixel_Sizes(face, size, size));
+    
+    DataFile::fontinfo_t fontinfo = {};
+    std::vector<DataFile::glyphentry_t> glyphtable;
+    std::vector<DataFile::dictentry_t> dictionary;
+   
+    // Convert size to pixels and round to nearest.
+    int u_per_em = face->units_per_EM;
+    auto topx = [size, u_per_em](int s) { return (s * size + u_per_em / 2) / u_per_em; };
+    
+    fontinfo.name = std::string(face->family_name) + " " +
+                    std::string(face->style_name) + " " +
+                    std::to_string(size);
+    
+    // Reserve 4 pixels on each side for antialiasing + hinting.
+    // They will be cropped off later.
+    fontinfo.max_width = topx(face->bbox.xMax - face->bbox.xMin) + 8;
+    fontinfo.max_height = topx(face->bbox.yMax - face->bbox.yMin) + 8;
+    fontinfo.baseline_x = topx(-face->bbox.xMin) + 4;
+    fontinfo.baseline_y = topx(face->bbox.yMax) + 4;
+    fontinfo.line_height = topx(face->height);
+    
+    FT_Int32 loadmode = FT_LOAD_TARGET_NORMAL | FT_LOAD_RENDER;
+    
+    if (bw)
+        loadmode = FT_LOAD_TARGET_MONO | FT_LOAD_MONOCHROME | FT_LOAD_RENDER;
+    
+    FT_ULong charcode;
+    FT_UInt gindex;
+    charcode = FT_Get_First_Char(face, &gindex);
+    while (gindex)
+    {
+        try
+        {
+            checkFT(FT_Load_Glyph(face, gindex, loadmode));
+        }
+        catch (std::runtime_error &e)
+        {
+            std::cerr << "Skipping glyph " << gindex << ": " << e.what() << std::endl;        
+            charcode = FT_Get_Next_Char(face, charcode, &gindex);
+        }
+        
+        DataFile::glyphentry_t glyph;
+        glyph.width = (face->glyph->advance.x + 32) / 64;
+        glyph.chars.push_back(charcode);
+        glyph.data.resize(fontinfo.max_width * fontinfo.max_height);
+        
+        int w = face->glyph->bitmap.width;
+        int dw = fontinfo.max_width;
+        int dx = fontinfo.baseline_x + face->glyph->bitmap_left;
+        int dy = fontinfo.baseline_y - face->glyph->bitmap_top;
+        
+        /* Some combining diacritics seem to exceed the bounding box.
+         * We don't support them all that well anyway, so just move
+         * them inside the box in order not to crash.. */
+        if (dy < 0)
+            dy = 0;
+        if (dy + face->glyph->bitmap.rows > fontinfo.max_height)
+            dy = fontinfo.max_height - face->glyph->bitmap.rows;
+        
+        size_t s = face->glyph->bitmap.pitch;
+        for (int y = 0; y < face->glyph->bitmap.rows; y++)
+        {
+            for (int x = 0; x < face->glyph->bitmap.width; x++)
+            {
+                size_t index = (y + dy) * dw + x + dx;
+                
+                if (face->glyph->bitmap.pixel_mode == FT_PIXEL_MODE_MONO)
+                {
+                    uint8_t byte = face->glyph->bitmap.buffer[s * y + x / 8];
+                    byte <<= x % 8;
+                    glyph.data.at(index) = (byte & 0x80) ? 15 : 0;
+                }
+                else
+                {
+                    glyph.data.at(index) =
+                        (face->glyph->bitmap.buffer[w * y + x] + 8) / 17;
+                }
+            }
+        }
+        glyphtable.push_back(glyph);
+        
+        charcode = FT_Get_Next_Char(face, charcode, &gindex);
+    }
+    
+    eliminate_duplicates(glyphtable);
+    crop_glyphs(glyphtable, fontinfo);
+    detect_flags(glyphtable, fontinfo);
+    
+    std::unique_ptr<DataFile> result(new DataFile(
+        dictionary, glyphtable, fontinfo));
+    return result;
+}
+
+}

tools/mcufontencoder/src/freetype_import.hh

@@ -0,0 +1,10 @@
+// Function for importing any font supported by libfreetype.
+
+#pragma once
+#include "datafile.hh"
+
+namespace mcufont {
+
+std::unique_ptr<DataFile> LoadFreetype(std::istream &file, int size, bool bw);
+
+}

tools/mcufontencoder/src/importtools.cc

@@ -0,0 +1,134 @@
+#include "importtools.hh"
+#include <limits>
+
+namespace mcufont {
+
+void eliminate_duplicates(std::vector<DataFile::glyphentry_t> &glyphtable)
+{
+    for (size_t i = 0; i + 1 < glyphtable.size(); i++)
+    {
+        for (size_t j = i + 1; j < glyphtable.size(); j++)
+        {
+            if (glyphtable.at(i).data == glyphtable.at(j).data &&
+                glyphtable.at(i).width == glyphtable.at(j).width)
+            {
+                for (int c : glyphtable.at(j).chars)
+                    glyphtable.at(i).chars.push_back(c);
+                
+                glyphtable.erase(glyphtable.begin() + j);
+                j--;
+            }
+        }
+    }
+}
+
+struct bbox_t
+{
+    int left;
+    int top;
+    int right;
+    int bottom;
+    
+    bbox_t()
+    {
+        left = std::numeric_limits<int>::max();
+        top = std::numeric_limits<int>::max();
+        right = std::numeric_limits<int>::min();
+        bottom = std::numeric_limits<int>::min();
+    }
+    
+    void update(int x, int y)
+    {
+        if (x < left) left = x;
+        if (x > right) right = x;
+        if (y < top) top = y;
+        if (y > bottom) bottom = y;
+    }
+};
+
+void crop_glyphs(std::vector<DataFile::glyphentry_t> &glyphtable,
+                 DataFile::fontinfo_t &fontinfo)
+{
+    // Find out the maximum bounding box
+    bbox_t bbox;
+    for (DataFile::glyphentry_t &glyph : glyphtable)
+    {
+        for (int y = 0; y < fontinfo.max_height; y++)
+        {
+            for (int x = 0; x < fontinfo.max_width; x++)
+            {
+                if (glyph.data.at(y * fontinfo.max_width + x))
+                    bbox.update(x, y);
+            }
+        }
+    }
+    
+    // Crop the glyphs to that
+    size_t old_w = fontinfo.max_width;
+    size_t new_w = bbox.right - bbox.left + 1;
+    size_t new_h = bbox.bottom - bbox.top + 1;
+    for (DataFile::glyphentry_t &glyph : glyphtable)
+    {
+        DataFile::pixels_t old = glyph.data;
+        glyph.data.clear();
+        
+        for (size_t y = 0; y < new_h; y++)
+        {
+            for (size_t x = 0; x < new_w; x++)
+            {
+                size_t old_x = bbox.left + x;
+                size_t old_y = bbox.top + y;
+                size_t old_pos = old_w * old_y + old_x; 
+                glyph.data.push_back(old.at(old_pos));
+            }
+        }
+    }
+    
+    fontinfo.max_width = new_w;
+    fontinfo.max_height = new_h;
+    fontinfo.baseline_x -= bbox.left;
+    fontinfo.baseline_y -= bbox.top;
+}
+
+void detect_flags(const std::vector<DataFile::glyphentry_t> &glyphtable,
+                  DataFile::fontinfo_t &fontinfo)
+{
+    if (!glyphtable.size())
+        return;
+    
+    // Check if all glyphs have equal width
+    int width = glyphtable[0].width;
+    bool is_monospace = true;
+    for (const DataFile::glyphentry_t &g : glyphtable)
+    {
+        if (g.width != width)
+        {
+            is_monospace = false;
+            break;
+        }
+    }
+    
+    if (is_monospace)
+        fontinfo.flags |= DataFile::FLAG_MONOSPACE;
+    
+    // Check if all glyphs contain only 0 or 15 alpha
+    bool is_bw = true;
+    for (const DataFile::glyphentry_t &g : glyphtable)
+    {
+        for (uint8_t pixel : g.data)
+        {
+            if (pixel != 0 && pixel != 15)
+            {
+                is_bw = false;
+                break;
+            }
+        }
+        if (!is_bw) break;
+    }
+    
+    if (is_bw)
+        fontinfo.flags |= DataFile::FLAG_BW;
+}
+
+
+}

tools/mcufontencoder/src/importtools.hh

@@ -0,0 +1,20 @@
+// Utility functions for processing imported font files.
+
+#pragma once
+#include "datafile.hh"
+
+namespace mcufont {
+
+// Find and eliminate any duplicate glyphs by appending their char vectors.
+void eliminate_duplicates(std::vector<DataFile::glyphentry_t> &glyphtable);
+
+// Calculate the maximum bounding box of the glyphs and crop them to that.
+// Adjust fontinfo accordingly.
+void crop_glyphs(std::vector<DataFile::glyphentry_t> &glyphtable,
+                 DataFile::fontinfo_t &fontinfo);
+
+// Fill in the flags (BW, monospace) automatically.
+void detect_flags(const std::vector<DataFile::glyphentry_t> &glyphtable,
+                  DataFile::fontinfo_t &fontinfo);
+
+}

tools/mcufontencoder/src/main.cc

@@ -0,0 +1,480 @@
+#include "datafile.hh"
+#include "importtools.hh"
+#include "bdf_import.hh"
+#include "freetype_import.hh"
+#include "export_rlefont.hh"
+#include "encode_rlefont.hh"
+#include "optimize_rlefont.hh"
+#include "export_bwfont.hh"
+#include <vector>
+#include <string>
+#include <set>
+#include <fstream>
+#include <iostream>
+#include <iomanip>
+#include <cstdlib>
+#include <ctime>
+#include <map>
+#include "ccfixes.hh"
+
+using namespace mcufont;
+
+static std::string strip_extension(std::string filename)
+{
+    size_t pos = filename.find_last_of('.');
+    
+    if (pos == std::string::npos)
+    {
+        return filename;
+    }
+    else
+    {
+        return filename.substr(0, pos);
+    }
+}
+
+static std::unique_ptr<DataFile> load_dat(std::string src)
+{
+    std::ifstream infile(src);
+    
+    if (!infile.good())
+    {
+        std::cerr << "Could not open " << src << std::endl;
+        return nullptr;
+    }
+    
+    std::unique_ptr<DataFile> f = DataFile::Load(infile);
+    if (!f)
+    {
+        std::cerr << "Invalid format for .dat file: " << src << std::endl;
+        return nullptr;
+    }
+    
+    return f;
+}
+
+static bool save_dat(std::string dest, DataFile *f)
+{
+    std::ofstream outfile(dest);
+    
+    if (!outfile.good())
+    {
+        std::cerr << "Could not open " << dest << std::endl;
+        return false;
+    }
+    
+    f->Save(outfile);
+    
+    if (!outfile.good())
+    {
+        std::cerr << "Could not write to " << dest << std::endl;
+        return false;
+    }
+    
+    return true;
+}
+
+enum status_t
+{
+    STATUS_OK = 0, // All good
+    STATUS_INVALID = 1, // Invalid command or args
+    STATUS_ERROR = 2 // Error when executing command
+};
+
+static status_t cmd_import_ttf(const std::vector<std::string> &args)
+{
+    if (args.size() != 3 && args.size() != 4)
+        return STATUS_INVALID;
+    
+    std::string src = args.at(1);
+    int size = std::stoi(args.at(2));
+    bool bw = (args.size() == 4 && args.at(3) == "bw");
+    std::string dest = strip_extension(src) + std::to_string(size) + (bw ? "bw" : "") + ".dat";
+    std::ifstream infile(src);
+    
+    if (!infile.good())
+    {
+        std::cerr << "Could not open " << src << std::endl;
+        return STATUS_ERROR;
+    }
+    
+    std::cout << "Importing " << src << " to " << dest << std::endl;
+    
+    std::unique_ptr<DataFile> f = LoadFreetype(infile, size, bw);
+    
+    mcufont::rlefont::init_dictionary(*f);
+    
+    if (!save_dat(dest, f.get()))
+        return STATUS_ERROR;
+    
+    std::cout << "Done: " << f->GetGlyphCount() << " unique glyphs." << std::endl;
+    return STATUS_OK;
+}
+
+static status_t cmd_import_bdf(const std::vector<std::string> &args)
+{
+    if (args.size() != 2)
+        return STATUS_INVALID;
+    
+    std::string src = args.at(1);
+    std::string dest = strip_extension(args.at(1)) + ".dat";
+    std::ifstream infile(src);
+    
+    if (!infile.good())
+    {
+        std::cerr << "Could not open " << src << std::endl;
+        return STATUS_ERROR;
+    }
+    
+    std::cout << "Importing " << src << " to " << dest << std::endl;
+    
+    std::unique_ptr<DataFile> f = LoadBDF(infile);
+    
+    mcufont::rlefont::init_dictionary(*f);
+    
+    if (!save_dat(dest, f.get()))
+        return STATUS_ERROR;
+    
+    std::cout << "Done: " << f->GetGlyphCount() << " unique glyphs." << std::endl;
+    return STATUS_OK;
+}
+
+static status_t cmd_filter(const std::vector<std::string> &args)
+{
+    if (args.size() < 3)
+        return STATUS_INVALID;
+    
+    std::set<int> allowed;
+    
+    // Parse arguments
+    for (size_t i = 2; i < args.size(); i++)
+    {
+        std::string s = args.at(i);
+        size_t pos = s.find('-');
+        if (pos == std::string::npos)
+        {
+            // Single char
+            allowed.insert(std::stoi(s, nullptr, 0));
+        }
+        else
+        {
+            // Range
+            int start = std::stoi(s.substr(0, pos), nullptr, 0);
+            int end = std::stoi(s.substr(pos + 1), nullptr, 0);
+            
+            for (int j = start; j <= end; j++)
+            {
+                allowed.insert(j);
+            }
+        }
+    }
+    
+    std::string src = args.at(1);
+    std::unique_ptr<DataFile> f = load_dat(src);
+    if (!f)
+        return STATUS_ERROR;
+    
+    std::cout << "Font originally had " << f->GetGlyphCount() << " glyphs." << std::endl;
+    
+    // Filter the glyphs
+    std::vector<DataFile::glyphentry_t> newglyphs;
+    for (size_t i = 0; i < f->GetGlyphCount(); i++)
+    {
+        DataFile::glyphentry_t g = f->GetGlyphEntry(i);
+        
+        for (size_t j = 0; j < g.chars.size(); j++)
+        {
+            if (!allowed.count(g.chars.at(j)))
+            {
+                g.chars.erase(g.chars.begin() + j);
+                j--;
+            }
+        }
+        
+        if (g.chars.size())
+        {
+            newglyphs.push_back(g);
+        }
+    }
+    
+    DataFile::fontinfo_t fontinfo = f->GetFontInfo();
+    crop_glyphs(newglyphs, fontinfo);
+    detect_flags(newglyphs, fontinfo);
+    
+    f.reset(new DataFile(f->GetDictionary(), newglyphs, fontinfo));
+    std::cout << "After filtering, " << f->GetGlyphCount() << " glyphs remain." << std::endl;
+    
+    if (!save_dat(src, f.get()))
+        return STATUS_ERROR;
+    
+    return STATUS_OK;
+}
+
+static status_t cmd_show_glyph(const std::vector<std::string> &args)
+{
+    if (args.size() != 3)
+        return STATUS_INVALID;
+    
+    std::string src = args.at(1);
+    std::unique_ptr<DataFile> f = load_dat(src);
+    
+    if (!f)
+        return STATUS_ERROR;
+    
+    size_t index = 0;
+    if (args.at(2) == "largest")
+    {
+        std::unique_ptr<mcufont::rlefont::encoded_font_t> e =
+            mcufont::rlefont::encode_font(*f, false);
+        size_t maxlen = 0;
+        size_t i = 0;
+        for (mcufont::rlefont::encoded_font_t::refstring_t g : e->glyphs)
+        {
+            if (g.size() > maxlen)
+            {
+                maxlen = g.size();
+                index = i;
+            }
+            i++;
+        }
+        
+        std::cout << "Index " << index << ", length " << maxlen << std::endl;
+    }
+    else
+    {
+        index = strtol(args.at(2).c_str(), nullptr, 0);
+    }
+    
+    if (index < 0 || index >= f->GetGlyphCount())
+    {
+        std::cerr << "No such glyph " << index << std::endl;
+        return STATUS_ERROR;
+    }
+    
+    std::cout << "Width: " << f->GetGlyphEntry(index).width << std::endl;
+    std::cout << "Chars: ";
+    for (int c: f->GetGlyphEntry(index).chars) std::cout << c << " ";
+    std::cout << std::endl;
+    
+    std::cout << f->GlyphToText(index);
+    return STATUS_OK;
+}
+
+static status_t cmd_rlefont_export(const std::vector<std::string> &args)
+{
+    if (args.size() != 2 && args.size() != 3)
+        return STATUS_INVALID;
+    
+    std::string src = args.at(1);
+    std::string dst = (args.size() == 2) ? strip_extension(src) + ".c" : args.at(2);
+    std::unique_ptr<DataFile> f = load_dat(src);
+    
+    if (!f)
+        return STATUS_ERROR;
+    
+    {
+        std::ofstream source(dst);
+        mcufont::rlefont::write_source(source, dst, *f);
+        std::cout << "Wrote " << dst << std::endl;
+    }
+    
+    return STATUS_OK;
+}
+
+static status_t cmd_rlefont_size(const std::vector<std::string> &args)
+{
+    if (args.size() != 2)
+        return STATUS_INVALID;
+    
+    std::string src = args.at(1);
+    std::unique_ptr<DataFile> f = load_dat(src);
+    
+    if (!f)
+        return STATUS_ERROR;
+    
+    size_t size = mcufont::rlefont::get_encoded_size(*f);
+    
+    std::cout << "Glyph count:       " << f->GetGlyphCount() << std::endl;
+    std::cout << "Glyph bbox:        " << f->GetFontInfo().max_width << "x"
+        << f->GetFontInfo().max_height << " pixels" << std::endl;
+    std::cout << "Uncompressed size: " << f->GetGlyphCount() *
+        f->GetFontInfo().max_width * f->GetFontInfo().max_height / 2
+        << " bytes" << std::endl;
+    std::cout << "Compressed size:   " << size << " bytes" << std::endl;
+    std::cout << "Bytes per glyph:   " << size / f->GetGlyphCount() << std::endl;
+    return STATUS_OK;
+}
+
+static status_t cmd_rlefont_optimize(const std::vector<std::string> &args)
+{
+    if (args.size() != 2 && args.size() != 3)
+        return STATUS_INVALID;
+    
+    std::string src = args.at(1);
+    std::unique_ptr<DataFile> f = load_dat(src);
+    
+    if (!f)
+        return STATUS_ERROR;
+    
+    size_t oldsize = mcufont::rlefont::get_encoded_size(*f);
+    
+    std::cout << "Original size is " << oldsize << " bytes" << std::endl;
+    std::cout << "Press ctrl-C at any time to stop." << std::endl;
+    std::cout << "Results are saved automatically after each iteration." << std::endl;
+    
+    int limit = 100;
+    if (args.size() == 3)
+    {
+        limit = std::stoi(args.at(2));
+    }
+    
+    if (limit > 0)
+        std::cout << "Limit is " << limit << " iterations" << std::endl;
+    
+    int i = 0;
+    time_t oldtime = time(NULL);
+    while (!limit || i < limit)
+    {
+        mcufont::rlefont::optimize(*f);
+
+        size_t newsize = mcufont::rlefont::get_encoded_size(*f);
+        time_t newtime = time(NULL);
+        
+        int bytes_per_min = (oldsize - newsize) * 60 / (newtime - oldtime + 1);
+        
+        i++;
+        std::cout << "iteration " << i << ", size " << newsize
+                  << " bytes, speed " << bytes_per_min << " B/min"
+                  << std::endl;
+        
+        {
+            if (!save_dat(src, f.get()))
+                return STATUS_ERROR;
+        }
+    }
+    
+    return STATUS_OK;
+}
+
+static status_t cmd_rlefont_show_encoded(const std::vector<std::string> &args)
+{
+    if (args.size() != 2)
+        return STATUS_INVALID;
+    
+    std::string src = args.at(1);
+    std::unique_ptr<DataFile> f = load_dat(src);
+    
+    if (!f)
+        return STATUS_ERROR;
+    
+    std::unique_ptr<mcufont::rlefont::encoded_font_t> e =
+        mcufont::rlefont::encode_font(*f, false);
+
+    int i = 0;
+    for (mcufont::rlefont::encoded_font_t::rlestring_t d : e->rle_dictionary)
+    {
+        std::cout << "Dict RLE " << 24 + i++ << ": ";
+        for (uint8_t v : d)
+            std::cout << std::setfill('0') << std::setw(2) << std::hex << (int)v << " ";
+        std::cout << std::endl;
+    }
+    
+    for (mcufont::rlefont::encoded_font_t::refstring_t d : e->ref_dictionary)
+    {
+        std::cout << "Dict Ref " << 24 + i++ << ": ";
+        for (uint8_t v : d)
+            std::cout << std::setfill('0') << std::setw(2) << std::hex << (int)v << " ";
+        std::cout << std::endl;
+    }
+    
+    i = 0;
+    for (mcufont::rlefont::encoded_font_t::refstring_t g : e->glyphs)
+    {
+        std::cout << "Glyph " << i++ << ": ";
+        for (uint8_t v : g)
+            std::cout << std::setfill('0') << std::setw(2) << std::hex << (int)v << " ";
+        std::cout << std::endl;
+    }
+    
+    return STATUS_OK;
+}
+
+static status_t cmd_bwfont_export(const std::vector<std::string> &args)
+{
+    if (args.size() != 2 && args.size() != 3)
+        return STATUS_INVALID;
+    
+    std::string src = args.at(1);
+    std::string dst = (args.size() == 2) ? strip_extension(src) + ".c" : args.at(2);
+    std::unique_ptr<DataFile> f = load_dat(src);
+    
+    if (!f)
+        return STATUS_ERROR;
+    
+    if (!(f->GetFontInfo().flags & DataFile::FLAG_BW))
+    {
+        std::cout << "Warning: font is not black and white" << std::endl;
+    }
+    
+    {
+        std::ofstream source(dst);
+        mcufont::bwfont::write_source(source, dst, *f);
+        std::cout << "Wrote " << dst << std::endl;
+    }
+    
+    return STATUS_OK;
+}
+
+
+static const char *usage_msg =
+    "Usage: mcufont <command> [options] ...\n"
+    "Commands for importing:\n"
+    "   import_ttf <ttffile> <size> [bw]     Import a .ttf font into a data file.\n"
+    "   import_bdf <bdffile>                 Import a .bdf font into a data file.\n"
+    "\n"
+    "Commands for inspecting and editing data files:\n"
+    "   filter <datfile> <range> ...         Remove everything except specified characters.\n"
+    "   show_glyph <datfile> <index>         Show the glyph at index.\n"
+    "\n"
+    "Commands specific to rlefont format:\n"
+    "   rlefont_size <datfile>               Check the encoded size of the data file.\n"
+    "   rlefont_optimize <datfile>           Perform an optimization pass on the data file.\n"
+    "   rlefont_export <datfile> [outfile]   Export to .c source code.\n"
+    "   rlefont_show_encoded <datfile>       Show the encoded data for debugging.\n"
+    "\n"
+    "Commands specific to bwfont format:\n"
+    "   bwfont_export <datfile> [outfile     Export to .c source code.\n"
+    "";
+
+typedef status_t (*cmd_t)(const std::vector<std::string> &args);
+static const std::map<std::string, cmd_t> command_list {
+    {"import_ttf",              cmd_import_ttf},
+    {"import_bdf",              cmd_import_bdf},
+    {"filter",                  cmd_filter},
+    {"show_glyph",              cmd_show_glyph},
+    {"rlefont_size",            cmd_rlefont_size},
+    {"rlefont_optimize",        cmd_rlefont_optimize},
+    {"rlefont_export",          cmd_rlefont_export},
+    {"rlefont_show_encoded",    cmd_rlefont_show_encoded},
+    {"bwfont_export",           cmd_bwfont_export},
+};
+
+int main(int argc, char **argv)
+{
+    std::vector<std::string> args;
+    for (int i = 1; i < argc; i++)
+        args.push_back(argv[i]);
+    
+    status_t status = STATUS_INVALID;
+    if (args.size() >= 1 && command_list.count(args.at(0)))
+    {
+        status = command_list.find(args.at(0))->second(args);
+    }
+    
+    if (status == STATUS_INVALID)
+    {
+        std::cout << usage_msg << std::endl;
+    }
+    
+    return status;
+}

tools/mcufontencoder/src/optimize_rlefont.cc

@@ -0,0 +1,417 @@
+#include "optimize_rlefont.hh"
+#include "encode_rlefont.hh"
+#include <random>
+#include <iostream>
+#include <set>
+#include <thread>
+#include <algorithm>
+#include "ccfixes.hh"
+
+namespace mcufont {
+namespace rlefont {
+
+typedef std::mt19937 rnd_t;
+
+// Select a random substring among all the glyphs in the datafile.
+std::unique_ptr<DataFile::pixels_t> random_substring(const DataFile &datafile, rnd_t &rnd)
+{
+    std::uniform_int_distribution<size_t> dist1(0, datafile.GetGlyphCount() - 1);
+    size_t index = dist1(rnd);
+    
+    const DataFile::pixels_t &pixels = datafile.GetGlyphEntry(index).data;
+    
+    std::uniform_int_distribution<size_t> dist2(2, pixels.size());
+    size_t length = dist2(rnd);
+    
+    std::uniform_int_distribution<size_t> dist3(0, pixels.size() - length);
+    size_t start = dist3(rnd);
+    
+    std::unique_ptr<DataFile::pixels_t> result;
+    result.reset(new DataFile::pixels_t(pixels.begin() + start,
+                                        pixels.begin() + start + length));
+    return result;
+}
+
+// Try to replace the worst dictionary entry with a better one.
+void optimize_worst(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose)
+{
+    std::uniform_int_distribution<size_t> dist(0, 1);
+    
+    DataFile trial = datafile;
+    size_t worst = trial.GetLowScoreIndex();
+    DataFile::dictentry_t d = trial.GetDictionaryEntry(worst);
+    d.replacement = *random_substring(datafile, rnd);
+    d.ref_encode = dist(rnd);
+    trial.SetDictionaryEntry(worst, d);
+    
+    size_t newsize = get_encoded_size(trial);
+    
+    if (newsize < size)
+    {
+        d.score = size - newsize;
+        datafile.SetDictionaryEntry(worst, d);
+        size = newsize;
+        
+        if (verbose)
+            std::cout << "optimize_worst: replaced " << worst
+                      << " score " << d.score << std::endl;
+    }
+}
+
+// Try to replace random dictionary entry with another one.
+void optimize_any(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose)
+{
+    DataFile trial = datafile;
+    std::uniform_int_distribution<size_t> dist(0, DataFile::dictionarysize - 1);
+    size_t index = dist(rnd);
+    DataFile::dictentry_t d = trial.GetDictionaryEntry(index);
+    d.replacement = *random_substring(datafile, rnd);
+    trial.SetDictionaryEntry(index, d);
+    
+    size_t newsize = get_encoded_size(trial);
+    
+    if (newsize < size)
+    {
+        d.score = size - newsize;
+        datafile.SetDictionaryEntry(index, d);
+        size = newsize;
+        
+        if (verbose)
+            std::cout << "optimize_any: replaced " << index
+                      << " score " << d.score << std::endl;
+    }
+}
+
+// Try to append or prepend random dictionary entry.
+void optimize_expand(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose, bool binary_only)
+{
+    DataFile trial = datafile;
+    std::uniform_int_distribution<size_t> dist1(0, DataFile::dictionarysize - 1);
+    size_t index = dist1(rnd);
+    DataFile::dictentry_t d = trial.GetDictionaryEntry(index);
+    
+    std::uniform_int_distribution<size_t> dist3(1, 3);
+    size_t count = dist3(rnd);
+    
+    for (size_t i = 0; i < count; i++)
+    {
+        std::uniform_int_distribution<size_t> booldist(0, 1);
+        std::uniform_int_distribution<size_t> pixeldist(0, 15);
+        uint8_t pixel;
+        
+        if (binary_only)
+        {
+            pixel = booldist(rnd) ? 15 : 0;
+        }
+        else
+        {
+            pixel = pixeldist(rnd);
+        }
+        
+        bool prepend = booldist(rnd);
+        
+        if (prepend)
+        {
+            d.replacement.insert(d.replacement.begin(), pixel);
+        }
+        else
+        {
+            d.replacement.push_back(pixel);
+        }
+    }
+    
+    trial.SetDictionaryEntry(index, d);
+    
+    size_t newsize = get_encoded_size(trial);
+    
+    if (newsize < size)
+    {
+        d.score = size - newsize;
+        datafile.SetDictionaryEntry(index, d);
+        size = newsize;
+        
+        if (verbose)
+            std::cout << "optimize_expand: expanded " << index
+                      << " by " << count << " pixels, score " << d.score << std::endl;
+    }
+}
+
+// Try to trim random dictionary entry.
+void optimize_trim(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose)
+{
+    DataFile trial = datafile;
+    std::uniform_int_distribution<size_t> dist1(0, DataFile::dictionarysize - 1);
+    size_t index = dist1(rnd);
+    DataFile::dictentry_t d = trial.GetDictionaryEntry(index);
+    
+    if (d.replacement.size() <= 2) return;
+    
+    std::uniform_int_distribution<size_t> dist2(0, std::min((int)d.replacement.size() / 2, 5));
+    size_t start = dist2(rnd);
+    size_t end = dist2(rnd);
+    
+    if (start)
+    {
+        d.replacement.erase(d.replacement.begin(), d.replacement.begin() + start);
+    }
+    
+    if (end)
+    {
+        d.replacement.erase(d.replacement.end() - end, d.replacement.end() - 1);
+    }
+    
+    trial.SetDictionaryEntry(index, d);
+    
+    size_t newsize = get_encoded_size(trial);
+    
+    if (newsize < size)
+    {
+        d.score = size - newsize;
+        datafile.SetDictionaryEntry(index, d);
+        size = newsize;
+        
+        if (verbose)
+            std::cout << "optimize_trim: trimmed " << index
+                      << " by " << start << " pixels from start and "
+                      << end << " pixels from end, score " << d.score << std::endl;
+    }
+}
+
+// Switch random dictionary entry to use ref encoding or back to rle.
+void optimize_refdict(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose)
+{
+    DataFile trial = datafile;
+    std::uniform_int_distribution<size_t> dist1(0, DataFile::dictionarysize - 1);
+    size_t index = dist1(rnd);
+    DataFile::dictentry_t d = trial.GetDictionaryEntry(index);
+    
+    d.ref_encode = !d.ref_encode;
+    
+    trial.SetDictionaryEntry(index, d);
+    
+    size_t newsize = get_encoded_size(trial);
+    
+    if (newsize < size)
+    {
+        d.score = size - newsize;
+        datafile.SetDictionaryEntry(index, d);
+        size = newsize;
+        
+        if (verbose)
+            std::cout << "optimize_refdict: switched " << index
+                      << " to " << (d.ref_encode ? "ref" : "RLE")
+                      << ", score " << d.score << std::endl;
+    }
+}
+
+// Combine two random dictionary entries.
+void optimize_combine(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose)
+{
+    DataFile trial = datafile;
+    std::uniform_int_distribution<size_t> dist1(0, DataFile::dictionarysize - 1);
+    size_t worst = datafile.GetLowScoreIndex();
+    size_t index1 = dist1(rnd);
+    size_t index2 = dist1(rnd);
+    
+    const DataFile::pixels_t &part1 = datafile.GetDictionaryEntry(index1).replacement;
+    const DataFile::pixels_t &part2 = datafile.GetDictionaryEntry(index2).replacement;
+    
+    DataFile::dictentry_t d;
+    d.replacement = part1;
+    d.replacement.insert(d.replacement.end(), part2.begin(), part2.end());
+    d.ref_encode = true;
+    trial.SetDictionaryEntry(worst, d);
+    
+    size_t newsize = get_encoded_size(trial);
+    
+    if (newsize < size)
+    {
+        d.score = size - newsize;
+        datafile.SetDictionaryEntry(worst, d);
+        size = newsize;
+        
+        if (verbose)
+            std::cout << "optimize_combine: combined " << index1
+                      << " and " << index2 << " to replace " << worst
+                      << ", score " << d.score << std::endl;
+    }
+}
+
+// Pick a random part of an encoded glyph and encode it as a ref dict.
+void optimize_encpart(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose)
+{
+    std::unique_ptr<encoded_font_t> e = encode_font(datafile);
+    
+    // Pick a random encoded glyph
+    std::uniform_int_distribution<size_t> dist1(0, datafile.GetGlyphCount() - 1);
+    size_t index = dist1(rnd);
+    const encoded_font_t::refstring_t &refstr = e->glyphs.at(index);
+    
+    if (refstr.size() < 2)
+        return;
+    
+    // Pick a random part of it
+    std::uniform_int_distribution<size_t> dist2(2, refstr.size());
+    size_t length = dist2(rnd);
+    std::uniform_int_distribution<size_t> dist3(0, refstr.size() - length);
+    size_t start = dist3(rnd);
+    
+    // Decode that part
+    encoded_font_t::refstring_t substr(refstr.begin() + start,
+                                       refstr.begin() + start + length);
+    std::unique_ptr<DataFile::pixels_t> decoded =
+        decode_glyph(*e, substr, datafile.GetFontInfo());
+    
+    // Add that as a new dictionary entry
+    DataFile trial = datafile;
+    size_t worst = trial.GetLowScoreIndex();
+    DataFile::dictentry_t d = trial.GetDictionaryEntry(worst);
+    d.replacement = *decoded;
+    d.ref_encode = true;
+    trial.SetDictionaryEntry(worst, d);
+    
+    size_t newsize = get_encoded_size(trial);
+    
+    if (newsize < size)
+    {
+        d.score = size - newsize;
+        datafile.SetDictionaryEntry(worst, d);
+        size = newsize;
+        
+        if (verbose)
+            std::cout << "optimize_encpart: replaced " << worst
+                      << " score " << d.score << std::endl;
+    }
+}
+
+// Execute all the optimization algorithms once.
+void optimize_pass(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose)
+{
+    optimize_worst(datafile, size, rnd, verbose);
+    optimize_any(datafile, size, rnd, verbose);
+    optimize_expand(datafile, size, rnd, verbose, false);
+    optimize_expand(datafile, size, rnd, verbose, true);
+    optimize_trim(datafile, size, rnd, verbose);
+    optimize_refdict(datafile, size, rnd, verbose);
+    optimize_combine(datafile, size, rnd, verbose);
+    optimize_encpart(datafile, size, rnd, verbose);
+}
+
+// Execute multiple passes in parallel and take the one with the best result.
+// The amount of parallelism is hardcoded in order to retain deterministic
+// behaviour.
+void optimize_parallel(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose, int num_threads = 4)
+{
+    std::vector<DataFile> datafiles;
+    std::vector<size_t> sizes;
+    std::vector<rnd_t> rnds;
+    std::vector<std::unique_ptr<std::thread> > threads;
+    
+    for (int i = 0; i < num_threads; i++)
+    {
+        datafiles.emplace_back(datafile);
+        sizes.emplace_back(size);
+        rnds.emplace_back(rnd());
+    }
+    
+    for (int i = 0; i < num_threads; i++)
+    {
+        threads.emplace_back(new std::thread(optimize_pass,
+                                             std::ref(datafiles.at(i)),
+                                             std::ref(sizes.at(i)),
+                                             std::ref(rnds.at(i)),
+                                             verbose));
+    }
+    
+    for (int i = 0; i < num_threads; i++)
+    {
+        threads.at(i)->join();
+    }
+    
+    int best = std::min_element(sizes.begin(), sizes.end()) - sizes.begin();
+    size = sizes.at(best);
+    datafile = datafiles.at(best);
+}
+
+// Go through all the dictionary entries and check what it costs to remove
+// them. Removes any entries with negative or zero score.
+void update_scores(DataFile &datafile, bool verbose)
+{
+    size_t oldsize = get_encoded_size(datafile);
+    
+    for (size_t i = 0; i < DataFile::dictionarysize; i++)
+    {
+        DataFile trial = datafile;
+        DataFile::dictentry_t dummy = {};
+        trial.SetDictionaryEntry(i, dummy);
+        size_t newsize = get_encoded_size(trial);
+        
+        DataFile::dictentry_t d = datafile.GetDictionaryEntry(i);
+        d.score = newsize - oldsize;
+        
+        if (d.score > 0)
+        {
+            datafile.SetDictionaryEntry(i, d);
+        }
+        else
+        {
+            datafile.SetDictionaryEntry(i, dummy);
+            
+            if (verbose && d.replacement.size() != 0)
+                std::cout << "update_scores: dropped " << i
+                        << " score " << -d.score << std::endl;
+        }
+    }
+}
+
+void init_dictionary(DataFile &datafile)
+{
+    rnd_t rnd(datafile.GetSeed());
+    
+    if (datafile.GetGlyphCount() == 0)
+        return;
+    
+    std::set<DataFile::pixels_t> seen_substrings;
+    std::set<DataFile::pixels_t> added_substrings;
+    
+    size_t i = 0;
+    while (i < DataFile::dictionarysize)
+    {
+        DataFile::pixels_t substring = *random_substring(datafile, rnd);
+        
+        if (!seen_substrings.count(substring))
+        {
+            seen_substrings.insert(substring);
+        }
+        else if (!added_substrings.count(substring))
+        {
+            // When we see a substring second time, add it.
+            DataFile::dictentry_t d;
+            d.score = 0;
+            d.replacement = substring;
+            datafile.SetDictionaryEntry(i, d);
+            i++;
+            added_substrings.insert(substring);
+        }
+    }
+}
+
+void optimize(DataFile &datafile, size_t iterations)
+{
+    bool verbose = false;
+    rnd_t rnd(datafile.GetSeed());
+    
+    update_scores(datafile, verbose);
+    
+    size_t size = get_encoded_size(datafile);
+    
+    for (size_t i = 0; i < iterations; i++)
+    {
+        optimize_parallel(datafile, size, rnd, verbose);
+    }
+    
+    std::uniform_int_distribution<size_t> dist(0, std::numeric_limits<uint32_t>::max());
+    datafile.SetSeed(dist(rnd));
+}
+
+}}

tools/mcufontencoder/src/optimize_rlefont.hh

@@ -0,0 +1,15 @@
+// This implements the actual optimization passes of the compressor.
+
+#include "datafile.hh"
+
+namespace mcufont {
+namespace rlefont {
+
+// Initialize the dictionary table with reasonable guesses.
+void init_dictionary(DataFile &datafile);
+
+// Perform a single optimization step, consisting itself of multiple passes
+// of each of the optimization algorithms.
+void optimize(DataFile &datafile, size_t iterations = 50);
+
+}}