diff --git a/3rdparty/freetype-2.6.1/readme.txt b/3rdparty/freetype-2.6.1/readme.txt new file mode 100644 index 00000000..797ce77d --- /dev/null +++ b/3rdparty/freetype-2.6.1/readme.txt @@ -0,0 +1,2 @@ +freetype-2.6.1 source code has not been included in this repository so that you are fully +aware of the license types and restrictions before using it with uGFX. diff --git a/tools/mcufontencoder/binaries/linux/mcufont b/tools/mcufontencoder/binaries/linux/mcufont new file mode 100644 index 00000000..d00acaf8 Binary files /dev/null and b/tools/mcufontencoder/binaries/linux/mcufont differ diff --git a/tools/mcufontencoder/binaries/windows/freetype6.dll b/tools/mcufontencoder/binaries/windows/freetype6.dll new file mode 100644 index 00000000..e35edc6b Binary files /dev/null and b/tools/mcufontencoder/binaries/windows/freetype6.dll differ diff --git a/tools/mcufontencoder/binaries/windows/libgcc_s_dw2-1.dll b/tools/mcufontencoder/binaries/windows/libgcc_s_dw2-1.dll new file mode 100644 index 00000000..99a76cf3 Binary files /dev/null and b/tools/mcufontencoder/binaries/windows/libgcc_s_dw2-1.dll differ diff --git a/tools/mcufontencoder/binaries/windows/mcufont.exe b/tools/mcufontencoder/binaries/windows/mcufont.exe new file mode 100644 index 00000000..40fe59cc Binary files /dev/null and b/tools/mcufontencoder/binaries/windows/mcufont.exe differ diff --git a/tools/mcufontencoder/binaries/windows/zlib1.dll b/tools/mcufontencoder/binaries/windows/zlib1.dll new file mode 100644 index 00000000..bb116109 Binary files /dev/null and b/tools/mcufontencoder/binaries/windows/zlib1.dll differ diff --git a/tools/mcufontencoder/src/Makefile b/tools/mcufontencoder/src/Makefile new file mode 100644 index 00000000..c5bfd929 --- /dev/null +++ b/tools/mcufontencoder/src/Makefile @@ -0,0 +1,43 @@ +CXXFLAGS = -O2 -Wall -Werror -Wno-unused-function -Wno-sign-compare -std=c++0x +CXXFLAGS += -ggdb +LDFLAGS += -pthread + +# Libfreetype +CXXFLAGS += $(shell freetype-config --cflags) +LDFLAGS += $(shell freetype-config --libs) + +# Class to represent font data internally +OBJS = datafile.o + +# Utility functions +OBJS += importtools.o exporttools.o + +# Import formats +OBJS += bdf_import.o freetype_import.o + +# rlefont export format +OBJS += encode_rlefont.o optimize_rlefont.o export_rlefont.o + +# bwfont export format +OBJS += export_bwfont.o + + +all: run_unittests mcufont + +clean: + rm -f unittests unittests.cc mcufont $(OBJS) + +mcufont: main.o $(OBJS) + g++ $(CXXFLAGS) -o $@ $^ $(LDFLAGS) + +unittests.cc: *.hh + cxxtestgen --have-eh --error-printer -o unittests.cc $^ + +unittests: unittests.cc $(OBJS) + g++ $(CXXFLAGS) -o $@ $^ $(LDFLAGS) + +run_unittests: unittests + ./unittests + +%.o: %.cc *.hh + g++ $(CXXFLAGS) -c $< diff --git a/tools/mcufontencoder/src/Makefile.mingw32 b/tools/mcufontencoder/src/Makefile.mingw32 new file mode 100644 index 00000000..e3eb4d0e --- /dev/null +++ b/tools/mcufontencoder/src/Makefile.mingw32 @@ -0,0 +1,44 @@ +ARCH = i686-pc-mingw32- +CXXFLAGS = -O2 -Wall -Werror -Wno-unused-function -Wno-sign-compare -std=c++0x +CXXFLAGS += -ggdb +LDFLAGS += -pthread --static + +# Libfreetype +CXXFLAGS += $(shell freetype-config --cflags) +LDFLAGS += $(shell freetype-config --libs) +#FREETYPE2_LIB = ../ugfx/3rdparty/freetype-2.6.1 +#CXXFLAGS += -I$(FREETYPE2_LIB)/include +#LDFLAGS += -I$(FREETYPE2_LIB)/lib -lfreetype + +# compiler fixes for mingw32 +CXXFLAGS += -DNEED_STRING_FIXES -DNEED_THREAD_FIXES + +# Class to represent font data internally +OBJS = datafile.o + +# Utility functions +OBJS += importtools.o exporttools.o + +# Import formats +OBJS += bdf_import.o freetype_import.o + +# rlefont export format +OBJS += encode_rlefont.o optimize_rlefont.o export_rlefont.o + +# bwfont export format +OBJS += export_bwfont.o + + +all: mcufont + +strip: mcufont.exe + strip mcufont.exe + +clean: + rm -f mcufont $(OBJS) + +mcufont: main.o $(OBJS) + $(ARCH)g++ $(CXXFLAGS) -o $@ $^ $(LDFLAGS) + +%.o: %.cc *.hh + $(ARCH)g++ $(CXXFLAGS) -c $< diff --git a/tools/mcufontencoder/src/bdf_import.cc b/tools/mcufontencoder/src/bdf_import.cc new file mode 100644 index 00000000..32deb057 --- /dev/null +++ b/tools/mcufontencoder/src/bdf_import.cc @@ -0,0 +1,156 @@ +#include "bdf_import.hh" +#include "importtools.hh" +#include +#include +#include +#include + +namespace mcufont { + +static std::string toupper(const std::string &input) +{ + std::string result; + for (char c: input) result.push_back(::toupper(c)); + return result; +} + +static int hextoint(char c) +{ + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + throw std::domain_error("Hex digit not in range"); +} + +static void parse_fontinfo(std::istream &file, DataFile::fontinfo_t &fontinfo) +{ + std::string line; + while (std::getline(file, line)) + { + std::istringstream s(line); + std::string tag; + s >> tag; + tag = toupper(tag); + + if (tag == "FONT") + { + while (isspace(s.peek())) s.get(); + std::getline(s, fontinfo.name); + } + else if (tag == "FONTBOUNDINGBOX") + { + int x, y; + s >> fontinfo.max_width >> fontinfo.max_height; + s >> x >> y; + fontinfo.baseline_x = - x; + fontinfo.baseline_y = fontinfo.max_height + y; + } + else if (tag == "STARTCHAR") + { + break; + } + } +} + +static bool parse_glyph(std::istream &file, DataFile::glyphentry_t &glyph, + const DataFile::fontinfo_t &fontinfo) +{ + glyph.chars.clear(); + glyph.width = 0; + + // Initialize the character contents to all 0 with proper size. + glyph.data.clear(); + glyph.data.resize(fontinfo.max_width * fontinfo.max_height, 0); + + int bbx_w = fontinfo.max_width; + int bbx_h = fontinfo.max_height; + int bbx_x = - fontinfo.baseline_x; + int bbx_y = fontinfo.baseline_y - fontinfo.max_height; + + // Read glyph metadata + std::string line; + std::string tag; + while (std::getline(file, line)) + { + std::istringstream s(line); + s >> tag; + tag = toupper(tag); + + if (tag == "ENCODING") + { + int c; + s >> c; + glyph.chars.push_back(c); + } + else if (tag == "DWIDTH") + { + s >> glyph.width; + } + else if (tag == "BBX") + { + s >> bbx_w >> bbx_h >> bbx_x >> bbx_y; + } + else if (tag == "BITMAP") + { + break; + } + } + + if (tag != "BITMAP") + return false; + + // Read glyph bits + int x0 = fontinfo.baseline_x + bbx_x; + int y = fontinfo.baseline_y - bbx_y - bbx_h; + for (int i = 0; i < bbx_h; i++) + { + std::getline(file, line); + line = toupper(line); + + for (int x = 0; x < bbx_w; x++) + { + int nibble = hextoint(line.at(x / 4)); + uint8_t pixel = 0; + if (nibble & (8 >> (x % 4))) + pixel = 15; + + glyph.data.at(y * fontinfo.max_width + x0 + x) = pixel; + } + + y++; + } + + std::getline(file, line); + line = toupper(line); + if (line.compare(0, 7, "ENDCHAR") == 0) + return true; + else + return false; +} + +std::unique_ptr LoadBDF(std::istream &file) +{ + DataFile::fontinfo_t fontinfo = {}; + std::vector glyphtable; + std::vector dictionary; + + parse_fontinfo(file, fontinfo); + + while (file) + { + DataFile::glyphentry_t glyph = {}; + if (parse_glyph(file, glyph, fontinfo)) + glyphtable.push_back(glyph); + } + + eliminate_duplicates(glyphtable); + crop_glyphs(glyphtable, fontinfo); + detect_flags(glyphtable, fontinfo); + + fontinfo.line_height = fontinfo.max_height; + + std::unique_ptr result(new DataFile( + dictionary, glyphtable, fontinfo)); + return result; +} + +} diff --git a/tools/mcufontencoder/src/bdf_import.hh b/tools/mcufontencoder/src/bdf_import.hh new file mode 100644 index 00000000..d94103e5 --- /dev/null +++ b/tools/mcufontencoder/src/bdf_import.hh @@ -0,0 +1,80 @@ +// Function for importing .BDF fonts as data files. + +#pragma once +#include "datafile.hh" + +namespace mcufont +{ + +std::unique_ptr LoadBDF(std::istream &file); + +} + +#ifdef CXXTEST_RUNNING +#include + +using namespace mcufont; + +class BDFTests: public CxxTest::TestSuite +{ +public: + void testLoadBDF() + { + std::istringstream s(testfile); + std::unique_ptr f = LoadBDF(s); + + TS_ASSERT_EQUALS(f->GetFontInfo().name, "-Misc-Fixed-Medium-R-Normal--14-130-75-75-C-70-ISO8859-15"); + TS_ASSERT_EQUALS(f->GetFontInfo().max_width, 6); + TS_ASSERT_EQUALS(f->GetFontInfo().max_height, 11); + TS_ASSERT_EQUALS(f->GetGlyphCount(), 1); + TS_ASSERT_EQUALS(f->GetGlyphEntry(0).chars.size(), 2); + } + +private: + static constexpr const char *testfile = + "STARTFONT 2.1\n" + "FONT -Misc-Fixed-Medium-R-Normal--14-130-75-75-C-70-ISO8859-15\n" + "FONTBOUNDINGBOX 7 14 0 -2\n" + "STARTCHAR defaultchar\n" + "ENCODING 0\n" + "DWIDTH 7 0\n" + "BBX 7 14 0 -2\n" + "BITMAP\n" + "00\n" + "B4\n" + "84\n" + "00\n" + "84\n" + "84\n" + "00\n" + "84\n" + "84\n" + "00\n" + "84\n" + "B4\n" + "00\n" + "00\n" + "ENDCHAR\n" + "STARTCHAR copychar\n" + "ENCODING 2\n" + "DWIDTH 7 0\n" + "BBX 7 14 0 -2\n" + "BITMAP\n" + "00\n" + "B4\n" + "84\n" + "00\n" + "84\n" + "84\n" + "00\n" + "84\n" + "84\n" + "00\n" + "84\n" + "B4\n" + "00\n" + "00\n" + "ENDCHAR\n"; + +}; +#endif diff --git a/tools/mcufontencoder/src/ccfixes.hh b/tools/mcufontencoder/src/ccfixes.hh new file mode 100644 index 00000000..8843639b --- /dev/null +++ b/tools/mcufontencoder/src/ccfixes.hh @@ -0,0 +1,152 @@ + +#ifdef NEED_STRING_FIXES + #include + #include + #include + #include + #include + + namespace std { + template inline std::string to_string(T value) + { + std::ostringstream os ; + os << value ; + return os.str() ; + } + + inline int stoi( const std::string& str, std::size_t* pos = 0, int base = 10 ) + { + const char* begin = str.c_str() ; + char* end = nullptr ; + long value = std::strtol( begin, &end, base ) ; + + if( errno == ERANGE || value > std::numeric_limits::max() ) + throw std::out_of_range( "stoi: out ofrange" ) ; + + if( end == str.c_str() ) + throw std::invalid_argument( "stoi: invalid argument" ) ; + + if(pos) *pos = end - begin ; + + return value ; + } + } +#endif + +#ifdef NEED_THREAD_FIXES +#ifndef WIN32STDTHREAD_H + #define WIN32STDTHREAD_H + + #include + #include + #include + #include + #include + #include + + #define _STD_THREAD_INVALID_HANDLE 0 + namespace std + { + + + class thread + { + public: + class id + { + DWORD mId; + void clear() {mId = 0;} + friend class thread; + public: + id(DWORD aId=0):mId(aId){} + bool operator==(const id& other) const {return mId == other.mId;} + }; + protected: + HANDLE mHandle; + id mThreadId; + public: + typedef HANDLE native_handle_type; + id get_id() const noexcept {return mThreadId;} + native_handle_type native_handle() const {return mHandle;} + thread(): mHandle(_STD_THREAD_INVALID_HANDLE){} + thread(thread& other) + :mHandle(other.mHandle), mThreadId(other.mThreadId) + { + other.mHandle = _STD_THREAD_INVALID_HANDLE; + other.mThreadId.clear(); + } + template + explicit thread(Function&& f, Args&&... args) + { + typedef decltype(std::bind(f, args...)) Call; + Call* call = new Call(std::bind(f, args...)); + mHandle = (HANDLE)_beginthreadex(NULL, 0, threadfunc, + (LPVOID)call, 0, (unsigned*)&(mThreadId.mId)); + } + template + static unsigned int __stdcall threadfunc(void* arg) + { + std::unique_ptr upCall(static_cast(arg)); + (*upCall)(); + return (unsigned long)0; + } + bool joinable() const {return mHandle != _STD_THREAD_INVALID_HANDLE;} + void join() + { + if (get_id() == GetCurrentThreadId()) + throw system_error(EDEADLK, generic_category()); + if (mHandle == _STD_THREAD_INVALID_HANDLE) + throw system_error(ESRCH, generic_category()); + if (!joinable()) + throw system_error(EINVAL, generic_category()); + WaitForSingleObject(mHandle, INFINITE); + CloseHandle(mHandle); + mHandle = _STD_THREAD_INVALID_HANDLE; + mThreadId.clear(); + } + + ~thread() + { + if (joinable()) + std::terminate(); + } + thread& operator=(const thread&) = delete; + thread& operator=(thread&& other) noexcept + { + if (joinable()) + std::terminate(); + swap(std::forward(other)); + return *this; + } + void swap(thread&& other) noexcept + { + std::swap(mHandle, other.mHandle); + std::swap(mThreadId.mId, other.mThreadId.mId); + } + static unsigned int hardware_concurrency() noexcept {return 1;} + void detach() + { + if (!joinable()) + throw system_error(); + mHandle = _STD_THREAD_INVALID_HANDLE; + mThreadId.clear(); + } + }; + namespace this_thread + { + inline thread::id get_id() {return thread::id(GetCurrentThreadId());} + inline void yield() {Sleep(0);} + template< class Rep, class Period > + void sleep_for( const std::chrono::duration& sleep_duration) + { + Sleep(chrono::duration_cast(sleep_duration).count()); + } + template + void sleep_until(const std::chrono::time_point& sleep_time) + { + sleep_for(sleep_time-Clock::now()); + } + } + } +#endif +#endif diff --git a/tools/mcufontencoder/src/datafile.cc b/tools/mcufontencoder/src/datafile.cc new file mode 100644 index 00000000..aba70057 --- /dev/null +++ b/tools/mcufontencoder/src/datafile.cc @@ -0,0 +1,239 @@ +#include "datafile.hh" +#include +#include +#include +#include +#include "ccfixes.hh" + +#define DATAFILE_FORMAT_VERSION 1 + +namespace mcufont { + +DataFile::DataFile(const std::vector &dictionary, + const std::vector &glyphs, + const fontinfo_t &fontinfo): + m_dictionary(dictionary), m_glyphtable(glyphs), m_fontinfo(fontinfo) +{ + dictentry_t dummy = {}; + while (m_dictionary.size() < dictionarysize) + m_dictionary.push_back(dummy); + + UpdateLowScoreIndex(); +} + +void DataFile::Save(std::ostream &file) const +{ + file << "Version " << DATAFILE_FORMAT_VERSION << std::endl; + file << "FontName " << m_fontinfo.name << std::endl; + file << "MaxWidth " << m_fontinfo.max_width << std::endl; + file << "MaxHeight " << m_fontinfo.max_height << std::endl; + file << "BaselineX " << m_fontinfo.baseline_x << std::endl; + file << "BaselineY " << m_fontinfo.baseline_y << std::endl; + file << "LineHeight " << m_fontinfo.line_height << std::endl; + file << "Flags " << m_fontinfo.flags << std::endl; + file << "RandomSeed " << m_seed << std::endl; + + for (const dictentry_t &d : m_dictionary) + { + if (d.replacement.size() != 0) + { + file << "DictEntry " << d.score << " "; + file << d.ref_encode << " " << d.replacement << std::endl; + } + } + + for (const glyphentry_t &g : m_glyphtable) + { + file << "Glyph "; + for (size_t i = 0; i < g.chars.size(); i++) + { + if (i != 0) file << ','; + file << g.chars.at(i); + } + file << " " << g.width << " " << g.data << std::endl; + } +} + +std::unique_ptr DataFile::Load(std::istream &file) +{ + fontinfo_t fontinfo = {}; + std::vector dictionary; + std::vector glyphtable; + uint32_t seed = 1234; + int version = -1; + + std::string line; + while (std::getline(file, line)) + { + std::istringstream input(line); + std::string tag; + + input >> tag; + + if (tag == "Version") + { + input >> version; + } + else if (tag == "FontName") + { + while (std::isspace(input.peek())) input.get(); + std::getline(input, fontinfo.name); + } + else if (tag == "MaxWidth") + { + input >> fontinfo.max_width; + } + else if (tag == "MaxHeight") + { + input >> fontinfo.max_height; + } + else if (tag == "BaselineX") + { + input >> fontinfo.baseline_x; + } + else if (tag == "BaselineY") + { + input >> fontinfo.baseline_y; + } + else if (tag == "LineHeight") + { + input >> fontinfo.line_height; + } + else if (tag == "RandomSeed") + { + input >> seed; + } + else if (tag == "Flags") + { + input >> fontinfo.flags; + } + else if (tag == "DictEntry" && dictionary.size() < dictionarysize) + { + dictentry_t d = {}; + input >> d.score >> d.ref_encode >> d.replacement; + dictionary.push_back(d); + } + else if (tag == "Glyph") + { + glyphentry_t g = {}; + std::string chars; + input >> chars >> g.width >> g.data; + + if ((int)g.data.size() != fontinfo.max_width * fontinfo.max_height) + throw std::runtime_error("wrong glyph data length: " + std::to_string(g.data.size())); + + size_t pos = 0; + while (pos < chars.size()) { + size_t p; + g.chars.push_back(std::stoi(chars.substr(pos), &p)); + pos += p + 1; + } + + glyphtable.push_back(g); + } + } + + if (version != DATAFILE_FORMAT_VERSION) + { + return std::unique_ptr(nullptr); + } + + std::unique_ptr result(new DataFile(dictionary, glyphtable, fontinfo)); + result->SetSeed(seed); + return result; +} + +void DataFile::SetDictionaryEntry(size_t index, const dictentry_t &value) +{ + m_dictionary.at(index) = value; + + if (index == m_lowscoreindex || + m_dictionary.at(m_lowscoreindex).score > value.score) + { + UpdateLowScoreIndex(); + } +} + +std::map DataFile::GetCharToGlyphMap() const +{ + std::map char_to_glyph; + + for (size_t i = 0; i < m_glyphtable.size(); i++) + { + for (size_t c: m_glyphtable[i].chars) + { + char_to_glyph[c] = i; + } + } + + return char_to_glyph; +} + +std::string DataFile::GlyphToText(size_t index) const +{ + std::ostringstream os; + + const char glyphchars[] = "....,,,,----XXXX"; + + for (int y = 0; y < m_fontinfo.max_height; y++) + { + for (int x = 0; x < m_fontinfo.max_width; x++) + { + size_t pos = y * m_fontinfo.max_width + x; + os << glyphchars[m_glyphtable.at(index).data.at(pos)]; + } + os << std::endl; + } + + return os.str(); +} + +void DataFile::UpdateLowScoreIndex() +{ + auto comparison = [](const dictentry_t &a, const dictentry_t &b) + { + return a.score < b.score; + }; + + auto iter = std::min_element(m_dictionary.begin(), + m_dictionary.end(), + comparison); + + m_lowscoreindex = iter - m_dictionary.begin(); +} + +std::ostream& operator<<(std::ostream& os, const DataFile::pixels_t& str) +{ + for (uint8_t p: str) + { + if (p <= 9) + os << (char)(p + '0'); + else if (p <= 15) + os << (char)(p - 10 + 'A'); + else + throw std::logic_error("invalid pixel alpha: " + std::to_string(p)); + } + return os; +} + +std::istream& operator>>(std::istream& is, DataFile::pixels_t& str) +{ + char c; + str.clear(); + + while (isspace(is.peek())) is.get(); + + while (is.get(c)) + { + if (c >= '0' && c <= '9') + str.push_back(c - '0'); + else if (c >= 'A' && c <= 'F') + str.push_back(c - 'A' + 10); + else + break; + } + + return is; +} + +} diff --git a/tools/mcufontencoder/src/datafile.hh b/tools/mcufontencoder/src/datafile.hh new file mode 100644 index 00000000..460e6039 --- /dev/null +++ b/tools/mcufontencoder/src/datafile.hh @@ -0,0 +1,174 @@ +// Class to store the data of a font while it is being processed. +// This class can be safely cloned using the default copy constructor. + +#pragma once +#include +#include +#include +#include +#include +#include + +namespace mcufont +{ + +class DataFile +{ +public: + typedef std::vector pixels_t; + + struct dictentry_t + { + pixels_t replacement; // The expanded version of this block. + int score; // Number of bytes that having this entry saves. + bool ref_encode; // Encode using references to other dictionary entries. + + dictentry_t(): score(0), ref_encode(false) {} + }; + + struct glyphentry_t + { + pixels_t data; // The full data of the glyph. + std::vector chars; // Characters that this glyph represents. + int width; // Tracking width of the character. + }; + + struct fontinfo_t + { + std::string name; // Name of the typeface + int max_width; // Width of the character bounding box. + int max_height; // Height of the character bounding box. + int baseline_x; // X coordinate (from left) of the baseline. + int baseline_y; // Y coordinate (from top) of the baseline. + int line_height; // Line height (vertical advance). + int flags; + }; + + static const int FLAG_MONOSPACE = 0x01; + static const int FLAG_BW = 0x02; + + // Construct from data in memory. + DataFile(const std::vector &dictionary, + const std::vector &glyphs, + const fontinfo_t &fontinfo); + + // Save to a file (custom format) + void Save(std::ostream &file) const; + + // Load from a file (custom format) + // Returns nullptr if load fails. + static std::unique_ptr Load(std::istream &file); + + // Get or set an entry in the dictionary. The size of the dictionary + // is constant. Entries 0 to 23 are reserved for special purposes. + static const size_t dictionarysize = 256 - 24; + const dictentry_t &GetDictionaryEntry(size_t index) const + { return m_dictionary.at(index); } + void SetDictionaryEntry(size_t index, const dictentry_t &value); + const std::vector &GetDictionary() const + { return m_dictionary; } + + // Get the index of the dictionary entry with the lowest score. + size_t GetLowScoreIndex() const + { return m_lowscoreindex; } + + // Get an entry in the glyph table. + size_t GetGlyphCount() const + { return m_glyphtable.size(); } + const glyphentry_t &GetGlyphEntry(size_t index) const + { return m_glyphtable.at(index); } + const std::vector &GetGlyphTable() const + { return m_glyphtable; } + + // Create a map of char indices to glyph indices + std::map GetCharToGlyphMap() const; + + // Get the information that applies to all glyphs. + const fontinfo_t &GetFontInfo() const + { return m_fontinfo; } + + // Show a glyph as text. + std::string GlyphToText(size_t index) const; + + // Get the random generator seed + // The seed is stored in the datafile to get deterministic behaviour + // for debugging. + uint32_t GetSeed() const { return m_seed; } + void SetSeed(uint32_t seed) { m_seed = seed; } + +private: + std::vector m_dictionary; + std::vector m_glyphtable; + fontinfo_t m_fontinfo; + uint32_t m_seed; + + size_t m_lowscoreindex; + + void UpdateLowScoreIndex(); +}; + +std::ostream& operator<<(std::ostream& os, const DataFile::pixels_t& str); +std::istream& operator>>(std::istream& is, DataFile::pixels_t& str); + +} + +#ifdef CXXTEST_RUNNING +#include + +using namespace mcufont; + +class DataFileTests: public CxxTest::TestSuite +{ +public: + void testFileLoad() + { + std::istringstream s(testfile); + std::unique_ptr f = DataFile::Load(s); + + TS_ASSERT_EQUALS(f->GetFontInfo().name, "Sans Serif"); + TS_ASSERT_EQUALS(f->GetFontInfo().max_width, 4); + TS_ASSERT_EQUALS(f->GetFontInfo().max_height, 6); + TS_ASSERT_EQUALS(f->GetDictionaryEntry(0).score, 5); + TS_ASSERT_EQUALS(f->GetDictionaryEntry(1).score, 13); + TS_ASSERT_EQUALS(f->GetGlyphCount(), 3); + + DataFile::pixels_t expected = { + 0,15,0,15, 0,15,0,15, 0,15,0,15, 0,15,0,15, 0,15,0,15, 0,15,0,15 + }; + TS_ASSERT_EQUALS(f->GetGlyphEntry(0).data.size(), 24); + TS_ASSERT(f->GetGlyphEntry(0).data == expected); + } + + void testFileSave() + { + std::istringstream is1(testfile); + std::unique_ptr f1 = DataFile::Load(is1); + + std::ostringstream os; + f1->Save(os); + + std::string text = os.str(); + std::istringstream is2(text); + std::unique_ptr f2 = DataFile::Load(is2); + + TS_ASSERT_EQUALS(f1->GetFontInfo().name, f2->GetFontInfo().name); + TS_ASSERT(f1->GetGlyphEntry(0).data == f2->GetGlyphEntry(0).data); + } + +private: + static constexpr const char *testfile = + "Version 1\n" + "FontName Sans Serif\n" + "MaxWidth 4\n" + "MaxHeight 6\n" + "BaselineX 1\n" + "BaselineY 1\n" + "DictEntry 5 0 0F0F0\n" + "DictEntry 13 0 F0F0F0\n" + "DictEntry 1 0 F0F0F0\n" + "Glyph 1,2,3 4 0F0F0F0F0F0F0F0F0F0F0F0F\n" + "Glyph 4 4 0F0F0F0F0F0F0F0F0F0F0F0F\n" + "Glyph 5 4 0F0F0F0F0F0F0F0F0F0F0F0F\n"; +}; + +#endif diff --git a/tools/mcufontencoder/src/encode_rlefont.cc b/tools/mcufontencoder/src/encode_rlefont.cc new file mode 100644 index 00000000..602f6033 --- /dev/null +++ b/tools/mcufontencoder/src/encode_rlefont.cc @@ -0,0 +1,735 @@ +#include "encode_rlefont.hh" +#include +#include +#include "ccfixes.hh" + +// Number of reserved codes before the dictionary entries. +#define DICT_START 24 + +// Special reference to mean "fill with zeros to the end of the glyph" +#define REF_FILLZEROS 16 + +// RLE codes +#define RLE_CODEMASK 0xC0 +#define RLE_VALMASK 0x3F +#define RLE_ZEROS 0x00 // 0 to 63 zeros +#define RLE_64ZEROS 0x40 // (1 to 64) * 64 zeros +#define RLE_ONES 0x80 // 1 to 64 full alphas +#define RLE_SHADE 0xC0 // 1 to 4 partial alphas + +// Dictionary "fill entries" for encoding bits directly. +#define DICT_START7BIT 4 +#define DICT_START6BIT 132 +#define DICT_START5BIT 196 +#define DICT_START4BIT 228 +#define DICT_START3BIT 244 +#define DICT_START2BIT 252 + +namespace mcufont { +namespace rlefont { + +// Get bit count for the "fill entries" +static size_t fillentry_bitcount(size_t index) +{ + if (index >= DICT_START2BIT) + return 2; + else if (index >= DICT_START3BIT) + return 3; + else if (index >= DICT_START4BIT) + return 4; + else if (index >= DICT_START5BIT) + return 5; + else if (index >= DICT_START6BIT) + return 6; + else + return 7; +} + +// Count the number of equal pixels at the beginning of the pixelstring. +static size_t prefix_length(const DataFile::pixels_t &pixels, size_t pos) +{ + uint8_t pixel = pixels.at(pos); + size_t count = 1; + while (pos + count < pixels.size() && + pixels.at(pos + count) == pixel) + { + count++; + } + return count; +} + +// Perform the RLE encoding for a dictionary entry. +static encoded_font_t::rlestring_t encode_rle(const DataFile::pixels_t &pixels) +{ + encoded_font_t::rlestring_t result; + + size_t pos = 0; + while (pos < pixels.size()) + { + uint8_t pixel = pixels.at(pos); + size_t count = prefix_length(pixels, pos); + pos += count; + + if (pixel == 0) + { + // Up to 63 zeros can be encoded with RLE_ZEROS. If there are more, + // encode using RLE_64ZEROS, and then whatever remains with RLE_ZEROS. + while (count >= 64) + { + size_t c = (count > 4096) ? 64 : (count / 64); + result.push_back(RLE_64ZEROS | (c - 1)); + count -= c * 64; + } + + if (count) + { + result.push_back(RLE_ZEROS | count); + } + } + else if (pixel == 15) + { + // Encode ones. + while (count) + { + size_t c = (count > 64) ? 64 : count; + result.push_back(RLE_ONES | (c - 1)); + count -= c; + } + } + else + { + // Encode shades. + while (count) + { + size_t c = (count > 4) ? 4 : count; + result.push_back(RLE_SHADE | ((c - 1) << 4) | pixel); + count -= c; + } + } + } + + return result; +} + +// We use a tree structure to represent the dictionary entries. +// Using this tree, we can perform a combined Aho-Corasick string matching +// and breadth-first search to find the optimal encoding of glyph data. +class DictTreeNode +{ +public: + constexpr DictTreeNode(): + m_index(-1), + m_ref(false), + m_length(0), + m_child0(nullptr), + m_child15(nullptr), + m_suffix(nullptr) + {} + + void SetChild(uint8_t p, DictTreeNode *child) + { + if (p == 0) + m_child0 = child; + else if (p == 15) + m_child15 = child; + else if (p > 15) + throw std::logic_error("invalid pixel alpha: " + std::to_string(p)); + else + { + if (!m_children) + { + m_children.reset(new DictTreeNode*[14]()); + } + m_children[p - 1] = child; + } + } + + DictTreeNode* GetChild(uint8_t p) const + { + if (p == 0) + return m_child0; + else if (p == 15) + return m_child15; + else if (p > 15) + throw std::logic_error("invalid pixel alpha: " + std::to_string(p)); + else if (!m_children) + return nullptr; + else + return m_children[p - 1]; + } + + bool HasIntermediateChildren() const { return m_children != nullptr; } + + int GetIndex() const { return m_index; } + void SetIndex(int index) { m_index = index; } + bool GetRef() const { return m_ref; } + void SetRef(bool ref) { m_ref = ref; } + size_t GetLength() const { return m_length; } + void SetLength(size_t length) { m_length = length; } + DictTreeNode *GetSuffix() const { return m_suffix; } + void SetSuffix(DictTreeNode *suffix) { m_suffix = suffix; } + +private: + // Index of dictionary entry or -1 if just a intermediate node. + int m_index; + + // True for ref-encoded dictionary entries. Used to avoid recursion when + // encoding them. + bool m_ref; + + // Length of the corresponding dictionary entry replacement. + // Equals the distance from the tree root. + size_t m_length; + + // Most tree nodes will only ever contains children for 0 or 15. + // Therefore the array for other nodes is allocated only on demand. + DictTreeNode *m_child0; + DictTreeNode *m_child15; + std::unique_ptr m_children; + + // Pointer to the longest suffix of this entry that exists in the + // dictionary. + DictTreeNode *m_suffix; +}; + +// Preallocated array for tree nodes +class TreeAllocator +{ +public: + TreeAllocator(size_t count) + { + m_storage.reset(new DictTreeNode[count]); + m_next = m_storage.get(); + m_left = count; + } + + DictTreeNode *allocate() + { + if (m_left == 0) + throw std::logic_error("Ran out of preallocated entries"); + + m_left--; + return m_next++; + } + +private: + std::unique_ptr m_storage; + DictTreeNode *m_next; + size_t m_left; +}; + +// Add a new dictionary entry to the tree. Adds the intermediate nodes, but +// does not yet fill the suffix pointers. +static DictTreeNode* add_tree_entry(const DataFile::pixels_t &entry, int index, + bool ref_encoded, DictTreeNode *root, + TreeAllocator &storage) +{ + DictTreeNode* node = root; + for (uint8_t p : entry) + { + DictTreeNode* branch = node->GetChild(p); + if (!branch) + { + branch = storage.allocate(); + node->SetChild(p, branch); + } + + node = branch; + } + + // Replace the entry if it either does not yet have an encoding, or if + // the new entry is non-ref (i.e. can be used in more situations). + if (node->GetIndex() < 0 || (node->GetRef() && !ref_encoded)) + { + node->SetIndex(index); + node->SetRef(ref_encoded); + node->SetLength(entry.size()); + } + + return node; +} + +// Walk the tree and find if the entry exists in the tree. If it does, +// returns a pointer to it, otherwise nullptr. +static DictTreeNode *find_tree_node(DataFile::pixels_t::const_iterator begin, + DataFile::pixels_t::const_iterator end, + DictTreeNode *root) +{ + DictTreeNode* node = root; + while (begin != end) + { + uint8_t pixel = *begin++; + node = node->GetChild(pixel); + + if (!node) + return nullptr; + } + + return node; +} + +// Fill in the suffix pointers recursively for the given subtree. +static void fill_tree_suffixes(DictTreeNode *root, DictTreeNode *subtree, + const DataFile::pixels_t &entry) +{ + for (size_t i = 1; i < entry.size(); i++) + { + DictTreeNode *node = find_tree_node(entry.begin() + i, entry.end(), root); + if (node) + { + subtree->SetSuffix(node); + break; + } + } + + if (!subtree->GetSuffix()) + subtree->SetSuffix(root); + + DataFile::pixels_t newentry(entry); + newentry.resize(entry.size() + 1); + for (uint8_t i = 0; i < 16; i++) + { + // Speed-up for the common case of 0 and 15 alphas. + if (i == 1 && !subtree->HasIntermediateChildren()) + i += 14; + + DictTreeNode *child = subtree->GetChild(i); + if (child) + { + newentry.at(entry.size()) = i; + fill_tree_suffixes(root, child, newentry); + } + } +} + +// Construct a lookup tree from the dictionary entries. +static DictTreeNode* construct_tree(const std::vector &dictionary, + TreeAllocator &storage, bool fast) +{ + DictTreeNode* root = storage.allocate(); + + // Populate the hardcoded entries for 0 to 15 alpha. + for (int j = 0; j < 16; j++) + { + DictTreeNode *node = storage.allocate(); + node->SetIndex(j); + node->SetRef(false); + node->SetLength(1); + root->SetChild(j, node); + } + + // Populate the actual dictionary entries + size_t i = DICT_START; + for (DataFile::dictentry_t d : dictionary) + { + if (!d.replacement.size()) + break; + + add_tree_entry(d.replacement, i, d.ref_encode, root, storage); + i++; + } + + if (!fast) + { + // Populate the fill entries for rest of dictionary + for (; i < 256; i++) + { + DataFile::pixels_t pixels; + size_t bitcount = fillentry_bitcount(i); + uint8_t byte = i - DICT_START7BIT; + for (size_t j = 0; j < bitcount; j++) + { + uint8_t p = (byte & (1 << j)) ? 15 : 0; + pixels.push_back(p); + } + + add_tree_entry(pixels, i, false, root, storage); + } + + // Fill in the suffix pointers for optimal encoding + DataFile::pixels_t nullentry; + fill_tree_suffixes(root, root, nullentry); + } + + return root; +} + +// Structure for keeping track of the shortest encoding to reach particular +// point of the pixel string. +struct encoding_link_t +{ + // Index of the position prior to the last dictionary entry. + size_t previous; + + // Index of the dictionary entry that brings us to this point. + int index; + + // Number of links to get here from the start of the string. + size_t length; + + constexpr encoding_link_t(): previous(0), index(-1), length(9999999) {} +}; + +// Perform the reference encoding for a glyph entry (optimal version). +// Uses a modified Aho-Corasick algorithm combined with breadth first search +// to find the shortest representation. +static encoded_font_t::refstring_t encode_ref_slow(const DataFile::pixels_t &pixels, + const DictTreeNode *root, + bool is_glyph) +{ + // Chain of encodings. Each entry in this array corresponds to a position + // in the pixel string. + std::unique_ptr chain(new encoding_link_t[pixels.size() + 1]); + + chain[0].previous = 0; + chain[0].index = 0; + chain[0].length = 0; + + // Read the pixels one-by-one and update the encoding links accordingly. + const DictTreeNode *node = root; + for (size_t pos = 0; pos < pixels.size(); pos++) + { + uint8_t pixel = pixels.at(pos); + const DictTreeNode *branch = node->GetChild(pixel); + + while (!branch) + { + // Cannot expand this sequence, defer to suffix. + node = node->GetSuffix(); + branch = node->GetChild(pixel); + } + + node = branch; + + // We have arrived at a new node, add it and any proper suffixes to + // the link chain. + const DictTreeNode *suffix = node; + while (suffix != root) + { + if (suffix->GetIndex() >= 0 && (is_glyph || !suffix->GetRef())) + { + encoding_link_t link; + link.previous = pos + 1 - suffix->GetLength(); + link.index = suffix->GetIndex(); + link.length = chain[link.previous].length + 1; + + if (link.length < chain[pos + 1].length) + chain[pos + 1] = link; + } + suffix = suffix->GetSuffix(); + } + } + + // Check if we can shorten the final encoding using REF_FILLZEROS. + if (is_glyph) + { + for (size_t pos = pixels.size() - 1; pos > 0; pos--) + { + if (pixels.at(pos) != 0) + break; + + encoding_link_t link; + link.previous = pos; + link.index = REF_FILLZEROS; + link.length = chain[pos].length + 1; + + if (link.length <= chain[pixels.size()].length) + chain[pixels.size()] = link; + } + } + + // Backtrack from the final link back to the start and construct the + // encoded string. + encoded_font_t::refstring_t result; + size_t len = chain[pixels.size()].length; + result.resize(len); + + size_t pos = pixels.size(); + for (size_t i = len; i > 0; i--) + { + result.at(i - 1) = chain[pos].index; + pos = chain[pos].previous; + } + + return result; +} + +// Walk the tree as far as possible following the given pixel string iterator. +// Returns number of pixels encoded, and index is set to the dictionary reference. +static size_t walk_tree(const DictTreeNode *tree, + DataFile::pixels_t::const_iterator pixels, + DataFile::pixels_t::const_iterator pixelsend, + int &index, bool is_glyph) +{ + size_t best_length = 0; + size_t length = 0; + index = -1; + + const DictTreeNode* node = tree; + while (pixels != pixelsend) + { + uint8_t pixel = *pixels++; + node = node->GetChild(pixel); + + if (!node) + break; + + length++; + + if (is_glyph || !node->GetRef()) + { + if (node->GetIndex() >= 0) + { + index = node->GetIndex(); + best_length = length; + } + } + } + + if (index < 0) + throw std::logic_error("walk_tree failed to find a valid encoding"); + + return best_length; +} + +// Perform the reference encoding for a glyph entry (fast version). +// Uses a simple greedy search to find select the encodings. +static encoded_font_t::refstring_t encode_ref_fast(const DataFile::pixels_t &pixels, + const DictTreeNode *tree, + bool is_glyph) +{ + encoded_font_t::refstring_t result; + + // Strip any zeroes from end + size_t end = pixels.size(); + + if (is_glyph) + { + while (end > 0 && pixels.at(end - 1) == 0) end--; + } + + size_t i = 0; + while (i < end) + { + int index; + i += walk_tree(tree, pixels.begin() + i, pixels.end(), index, is_glyph); + result.push_back(index); + } + + if (i < pixels.size()) + result.push_back(REF_FILLZEROS); + + return result; +} + +static encoded_font_t::refstring_t encode_ref(const DataFile::pixels_t &pixels, + const DictTreeNode *tree, + bool is_glyph, bool fast) +{ + if (fast) + return encode_ref_fast(pixels, tree, is_glyph); + else + return encode_ref_slow(pixels, tree, is_glyph); +} + +// Compare dictionary entries by their coding type. +// Sorts RLE-encoded entries first and any empty entries last. +static bool cmp_dict_coding(const DataFile::dictentry_t &a, + const DataFile::dictentry_t &b) +{ + if (a.replacement.size() == 0 && b.replacement.size() != 0) + return false; + else if (a.replacement.size() != 0 && b.replacement.size() == 0) + return true; + else if (a.ref_encode == false && b.ref_encode == true) + return true; + else + return false; +} + +size_t estimate_tree_node_count(const std::vector &dict) +{ + size_t count = DICT_START; // Preallocated entries + for (const DataFile::dictentry_t &d: dict) + { + count += d.replacement.size(); + } + count += 128 * 7; // Fill entries + return count; +} + +std::unique_ptr encode_font(const DataFile &datafile, + bool fast) +{ + std::unique_ptr result(new encoded_font_t); + + // Sort the dictionary so that RLE-coded entries come first. + // This way the two are easy to distinguish based on index. + std::vector sorted_dict = datafile.GetDictionary(); + std::stable_sort(sorted_dict.begin(), sorted_dict.end(), cmp_dict_coding); + + // Build the binary tree for looking up references. + size_t count = estimate_tree_node_count(sorted_dict); + TreeAllocator allocator(count); + DictTreeNode* tree = construct_tree(sorted_dict, allocator, fast); + + // Encode the dictionary entries, using either RLE or reference method. + for (const DataFile::dictentry_t &d : sorted_dict) + { + if (d.replacement.size() == 0) + { + continue; + } + else if (d.ref_encode) + { + result->ref_dictionary.push_back(encode_ref(d.replacement, tree, false, fast)); + } + else + { + result->rle_dictionary.push_back(encode_rle(d.replacement)); + } + } + + // Then reference-encode the glyphs + for (const DataFile::glyphentry_t &g : datafile.GetGlyphTable()) + { + result->glyphs.push_back(encode_ref(g.data, tree, true, fast)); + } + + // Optionally verify that the encoding was correct. + if (!fast) + { + for (size_t i = 0; i < datafile.GetGlyphCount(); i++) + { + std::unique_ptr decoded = + decode_glyph(*result, i, datafile.GetFontInfo()); + if (*decoded != datafile.GetGlyphEntry(i).data) + { + auto iter = std::mismatch(decoded->begin(), decoded->end(), + datafile.GetGlyphEntry(i).data.begin()); + size_t pos = iter.first - decoded->begin(); + throw std::logic_error("verification of glyph " + std::to_string(i) + + " failed at position " + std::to_string(pos)); + } + } + } + + return result; +} + +size_t get_encoded_size(const encoded_font_t &encoded) +{ + size_t total = 0; + for (const encoded_font_t::rlestring_t &r : encoded.rle_dictionary) + { + total += r.size(); + + if (r.size() != 0) + total += 2; // Offset table entry + } + for (const encoded_font_t::refstring_t &r : encoded.ref_dictionary) + { + total += r.size(); + + if (r.size() != 0) + total += 2; // Offset table entry + } + for (const encoded_font_t::refstring_t &r : encoded.glyphs) + { + total += r.size(); + total += 2; // Offset table entry + total += 1; // Width table entry + } + return total; +} + +std::unique_ptr decode_glyph( + const encoded_font_t &encoded, + const encoded_font_t::refstring_t &refstring, + const DataFile::fontinfo_t &fontinfo) +{ + std::unique_ptr result(new DataFile::pixels_t); + + for (uint8_t ref : refstring) + { + if (ref <= 15) + { + result->push_back(ref); + } + else if (ref == REF_FILLZEROS) + { + result->resize(fontinfo.max_width * fontinfo.max_height, 0); + } + else if (ref < DICT_START) + { + throw std::logic_error("unknown code: " + std::to_string(ref)); + } + else if (ref - DICT_START < (int)encoded.rle_dictionary.size()) + { + for (uint8_t rle : encoded.rle_dictionary.at(ref - DICT_START)) + { + if ((rle & RLE_CODEMASK) == RLE_ZEROS) + { + for (int i = 0; i < (rle & RLE_VALMASK); i++) + { + result->push_back(0); + } + } + else if ((rle & RLE_CODEMASK) == RLE_64ZEROS) + { + for (int i = 0; i < ((rle & RLE_VALMASK) + 1) * 64; i++) + { + result->push_back(0); + } + } + else if ((rle & RLE_CODEMASK) == RLE_ONES) + { + for (int i = 0; i < (rle & RLE_VALMASK) + 1; i++) + { + result->push_back(15); + } + } + else if ((rle & RLE_CODEMASK) == RLE_SHADE) + { + uint8_t count, alpha; + count = ((rle & RLE_VALMASK) >> 4) + 1; + alpha = ((rle & RLE_VALMASK) & 0xF); + for (int i = 0; i < count; i++) + { + result->push_back(alpha); + } + } + } + } + else if (ref - DICT_START - encoded.rle_dictionary.size() < encoded.ref_dictionary.size()) + { + size_t index = ref - DICT_START - encoded.rle_dictionary.size(); + std::unique_ptr part = + decode_glyph(encoded, encoded.ref_dictionary.at(index), + fontinfo); + result->insert(result->end(), part->begin(), part->end()); + } + else + { + size_t bitcount = fillentry_bitcount(ref); + + uint8_t byte = ref - DICT_START7BIT; + for (size_t i = 0; i < bitcount; i++) + { + uint8_t p = (byte & (1 << i)) ? 15 : 0; + result->push_back(p); + } + } + } + + return result; +} + +std::unique_ptr decode_glyph( + const encoded_font_t &encoded, size_t index, + const DataFile::fontinfo_t &fontinfo) +{ + return decode_glyph(encoded, encoded.glyphs.at(index), fontinfo); +} + +}} diff --git a/tools/mcufontencoder/src/encode_rlefont.hh b/tools/mcufontencoder/src/encode_rlefont.hh new file mode 100644 index 00000000..82e72523 --- /dev/null +++ b/tools/mcufontencoder/src/encode_rlefont.hh @@ -0,0 +1,126 @@ +// Given a dictionary and glyphs, encode the data for all the glyphs. + +#pragma once + +#include "datafile.hh" +#include +#include + +namespace mcufont { +namespace rlefont { + +struct encoded_font_t +{ + // RLE-encoded format for storing the dictionary entries. + // Each item is a byte. Top bit means the value in the original bitstream, + // and the bottom 7 bits store the repetition count. + typedef std::vector rlestring_t; + + // Reference encoded format for storing the glyphs. + // Each item is a reference to the dictionary. + // Values 0 and 1 are hardcoded to mean 0 and 1. + // All other values mean dictionary entry at (i-2). + typedef std::vector refstring_t; + + std::vector rle_dictionary; + std::vector ref_dictionary; + std::vector glyphs; +}; + +// Encode all the glyphs. +std::unique_ptr encode_font(const DataFile &datafile, + bool fast = true); + +// Sum up the total size of the encoded glyphs + dictionary. +size_t get_encoded_size(const encoded_font_t &encoded); + +inline size_t get_encoded_size(const DataFile &datafile, bool fast = true) +{ + std::unique_ptr e = encode_font(datafile, fast); + return get_encoded_size(*e); +} + +// Decode a single glyph (for verification). +std::unique_ptr decode_glyph( + const encoded_font_t &encoded, + const encoded_font_t::refstring_t &refstring, + const DataFile::fontinfo_t &fontinfo); + +// Decode a single glyph (for verification). +std::unique_ptr decode_glyph( + const encoded_font_t &encoded, size_t index, + const DataFile::fontinfo_t &fontinfo); + +}} + + +#ifdef CXXTEST_RUNNING +#include + +using namespace mcufont; +using namespace mcufont::rlefont; + +class RLEFontEncodeTests: public CxxTest::TestSuite +{ +public: + void testEncode() + { + std::istringstream s(testfile); + std::unique_ptr f = DataFile::Load(s); + std::unique_ptr e = encode_font(*f, false); + + TS_ASSERT_EQUALS(e->glyphs.size(), 3); + + // Expected values for dictionary + encoded_font_t::rlestring_t dict0 = {0x01, 0xCE, 0x01, 0xCE}; + encoded_font_t::rlestring_t dict1 = {0x0C}; + encoded_font_t::rlestring_t dict2 = {0xFE}; + encoded_font_t::refstring_t dict3 = {24, 24}; + + TS_ASSERT(e->rle_dictionary.at(0) == dict0); + TS_ASSERT(e->rle_dictionary.at(1) == dict1); + TS_ASSERT(e->rle_dictionary.at(2) == dict2); + TS_ASSERT(e->ref_dictionary.at(0) == dict3); + + // Expected values for glyphs + encoded_font_t::refstring_t glyph0 = {27, 27, 27}; + encoded_font_t::refstring_t glyph1 = {24, 0, 132, 25, 14}; + encoded_font_t::refstring_t glyph2 = {228, 26, 244, 14, 14, 14, 228, 26, 16}; + + TS_ASSERT_EQUALS(e->glyphs.at(0), glyph0); + TS_ASSERT_EQUALS(e->glyphs.at(1), glyph1); + TS_ASSERT_EQUALS(e->glyphs.at(2), glyph2); + } + + void testDecode() + { + std::istringstream s(testfile); + std::unique_ptr f = DataFile::Load(s); + std::unique_ptr e = encode_font(*f, false); + + for (size_t i = 0; i < 3; i++) + { + std::unique_ptr dec; + dec = decode_glyph(*e, i, f->GetFontInfo()); + + TS_ASSERT_EQUALS(*dec, f->GetGlyphEntry(i).data); + } + } + +private: + static constexpr const char *testfile = + "Version 1\n" + "FontName Sans Serif\n" + "MaxWidth 4\n" + "MaxHeight 6\n" + "BaselineX 1\n" + "BaselineY 1\n" + "DictEntry 1 0 0E0E\n" + "DictEntry 1 0 000000000000\n" + "DictEntry 1 0 EEEE\n" + "DictEntry 1 1 0E0E0E0E\n" + "Glyph 0 4 0E0E0E0E0E0E0E0E0E0E0E0E\n" + "Glyph 1 4 0E0E0000000000000000000E\n" + "Glyph 2 4 0000EEEE000EEE0000EEEE00\n"; +}; +#endif diff --git a/tools/mcufontencoder/src/export_bwfont.cc b/tools/mcufontencoder/src/export_bwfont.cc new file mode 100644 index 00000000..5e9a6779 --- /dev/null +++ b/tools/mcufontencoder/src/export_bwfont.cc @@ -0,0 +1,247 @@ +#include "export_bwfont.hh" +#include +#include +#include +#include +#include +#include +#include +#include "exporttools.hh" +#include "importtools.hh" +#include "ccfixes.hh" + +#define BWFONT_FORMAT_VERSION 4 + +namespace mcufont { +namespace bwfont { + +static void encode_glyph(const DataFile::glyphentry_t &glyph, + const DataFile::fontinfo_t &fontinfo, + std::vector &dest, + int num_cols) +{ + const int threshold = 8; + + // Find the number of columns in the glyph data + if (num_cols == 0) + { + for (int x = 0; x < fontinfo.max_width; x++) + { + for (int y = 0; y < fontinfo.max_height; y++) + { + size_t index = y * fontinfo.max_width + x; + if (glyph.data.at(index) >= threshold) + num_cols = x + 1; + } + } + } + + // Write the bits that compose the glyph + for (int x = 0; x < num_cols; x++) + { + for (int y = 0; y < fontinfo.max_height; y+= 8) + { + size_t remain = std::min(8, fontinfo.max_height - y); + uint8_t byte = 0; + for (size_t i = 0; i < remain; i++) + { + size_t index = (y + i) * fontinfo.max_width + x; + if (glyph.data.at(index) >= threshold) + { + byte |= (1 << i); + } + } + dest.push_back(byte); + } + } +} + +struct cropinfo_t +{ + size_t offset_x; + size_t offset_y; + size_t height_bytes; + size_t height_pixels; + size_t width; +}; + +static void encode_character_range(std::ostream &out, + const std::string &name, + const DataFile &datafile, + const char_range_t &range, + unsigned range_index, + cropinfo_t &cropinfo) +{ + std::vector glyphs; + bool constant_width = true; + int width = datafile.GetGlyphEntry(range.glyph_indices[0]).width; + + // Copy all the glyphs in this range for the purpose of cropping them. + for (int glyph_index: range.glyph_indices) + { + if (glyph_index < 0) + { + // Missing glyph + DataFile::glyphentry_t dummy = {}; + glyphs.push_back(dummy); + } + else + { + auto glyph = datafile.GetGlyphEntry(glyph_index); + glyphs.push_back(glyph); + + if (glyph.width != width) + { + constant_width = false; + width = 0; + } + } + } + + // Crop the glyphs in this range. Getting rid of a few rows at top + // or left can save a bunch of bytes with minimal cost. + DataFile::fontinfo_t old_fi = datafile.GetFontInfo(); + DataFile::fontinfo_t new_fi = old_fi; + crop_glyphs(glyphs, new_fi); + + if (new_fi.max_width != width) + { + constant_width = false; + width = 0; + } + + // Fill in the crop information + cropinfo.offset_x = old_fi.baseline_x - new_fi.baseline_x; + cropinfo.offset_y = old_fi.baseline_y - new_fi.baseline_y; + cropinfo.height_pixels = new_fi.max_height; + cropinfo.height_bytes = (cropinfo.height_pixels + 7) / 8; + cropinfo.width = width; + + // Then format and write out the glyph data + std::vector offsets; + std::vector data; + std::vector widths; + size_t stride = cropinfo.height_bytes; + + for (const DataFile::glyphentry_t &g : glyphs) + { + offsets.push_back(data.size() / stride); + widths.push_back(g.width); + encode_glyph(g, new_fi, data, width); + } + offsets.push_back(data.size() / stride); + + write_const_table(out, data, "uint8_t", "mf_bwfont_" + name + "_glyph_data_" + std::to_string(range_index)); + + if (!constant_width) + { + write_const_table(out, offsets, "uint16_t", "mf_bwfont_" + name + "_glyph_offsets_" + std::to_string(range_index), 4); + write_const_table(out, widths, "uint8_t", "mf_bwfont_" + name + "_glyph_widths_" + std::to_string(range_index)); + } +} + +void write_source(std::ostream &out, std::string name, const DataFile &datafile) +{ + name = filename_to_identifier(name); + + out << std::endl; + out << std::endl; + out << "/* Start of automatically generated font definition for " << name << ". */" << std::endl; + out << std::endl; + + out << "#ifndef MF_BWFONT_INTERNALS" << std::endl; + out << "#define MF_BWFONT_INTERNALS" << std::endl; + out << "#endif" << std::endl; + out << "#include \"mf_bwfont.h\"" << std::endl; + out << std::endl; + + out << "#ifndef MF_BWFONT_VERSION_" << BWFONT_FORMAT_VERSION << "_SUPPORTED" << std::endl; + out << "#error The font file is not compatible with this version of mcufont." << std::endl; + out << "#endif" << std::endl; + out << std::endl; + + // Split the characters into ranges + DataFile::fontinfo_t f = datafile.GetFontInfo(); + size_t glyph_size = f.max_width * ((f.max_height + 7) / 8); + auto get_glyph_size = [=](size_t i) { return glyph_size; }; + std::vector ranges = compute_char_ranges(datafile, + get_glyph_size, 65536, 16); + + // Write out glyph data for character ranges + std::vector crops; + for (size_t i = 0; i < ranges.size(); i++) + { + cropinfo_t cropinfo; + encode_character_range(out, name, datafile, ranges.at(i), i, cropinfo); + crops.push_back(cropinfo); + } + + // Write out a table describing the character ranges + out << "static const struct mf_bwfont_char_range_s mf_bwfont_" + name + "_char_ranges[] = {" << std::endl; + for (size_t i = 0; i < ranges.size(); i++) + { + std::string offsets = (crops[i].width) ? "0" : "mf_bwfont_" + name + "_glyph_offsets_" + std::to_string(i); + std::string widths = (crops[i].width) ? "0" : "mf_bwfont_" + name + "_glyph_widths_" + std::to_string(i); + + out << " {" << std::endl; + out << " " << ranges.at(i).first_char << ", /* first char */" << std::endl; + out << " " << ranges.at(i).char_count << ", /* char count */" << std::endl; + out << " " << crops[i].offset_x << ", /* offset x */" << std::endl; + out << " " << crops[i].offset_y << ", /* offset y */" << std::endl; + out << " " << crops[i].height_bytes << ", /* height in bytes */" << std::endl; + out << " " << crops[i].height_pixels << ", /* height in pixels */" << std::endl; + out << " " << crops[i].width << ", /* width */" << std::endl; + out << " " << widths << ", /* glyph widths */" << std::endl; + out << " " << offsets << ", /* glyph offsets */" << std::endl; + out << " " << "mf_bwfont_" << name << "_glyph_data_" << i << ", /* glyph data */" << std::endl; + out << " }," << std::endl; + } + out << "};" << std::endl; + out << std::endl; + + // Fonts in this format are always black & white + int flags = datafile.GetFontInfo().flags | DataFile::FLAG_BW; + + // Pull it all together in the rlefont_s structure. + out << "const struct mf_bwfont_s mf_bwfont_" << name << " = {" << std::endl; + out << " {" << std::endl; + out << " " << "\"" << datafile.GetFontInfo().name << "\"," << std::endl; + out << " " << "\"" << name << "\"," << std::endl; + out << " " << datafile.GetFontInfo().max_width << ", /* width */" << std::endl; + out << " " << datafile.GetFontInfo().max_height << ", /* height */" << std::endl; + out << " " << get_min_x_advance(datafile) << ", /* min x advance */" << std::endl; + out << " " << get_max_x_advance(datafile) << ", /* max x advance */" << std::endl; + out << " " << datafile.GetFontInfo().baseline_x << ", /* baseline x */" << std::endl; + out << " " << datafile.GetFontInfo().baseline_y << ", /* baseline y */" << std::endl; + out << " " << datafile.GetFontInfo().line_height << ", /* line height */" << std::endl; + out << " " << flags << ", /* flags */" << std::endl; + out << " " << select_fallback_char(datafile) << ", /* fallback character */" << std::endl; + out << " " << "&mf_bwfont_character_width," << std::endl; + out << " " << "&mf_bwfont_render_character," << std::endl; + out << " }," << std::endl; + + out << " " << BWFONT_FORMAT_VERSION << ", /* version */" << std::endl; + out << " " << ranges.size() << ", /* char range count */" << std::endl; + out << " " << "mf_bwfont_" << name << "_char_ranges," << std::endl; + out << "};" << std::endl; + + // Write the font lookup structure + out << std::endl; + out << "#ifdef MF_INCLUDED_FONTS" << std::endl; + out << "/* List entry for searching fonts by name. */" << std::endl; + out << "static const struct mf_font_list_s mf_bwfont_" << name << "_listentry = {" << std::endl; + out << " MF_INCLUDED_FONTS," << std::endl; + out << " (struct mf_font_s*)&mf_bwfont_" << name << std::endl; + out << "};" << std::endl; + out << "#undef MF_INCLUDED_FONTS" << std::endl; + out << "#define MF_INCLUDED_FONTS (&mf_bwfont_" << name << "_listentry)" << std::endl; + out << "#endif" << std::endl; + + out << std::endl; + out << std::endl; + out << "/* End of automatically generated font definition for " << name << ". */" << std::endl; + out << std::endl; +} + + +}} diff --git a/tools/mcufontencoder/src/export_bwfont.hh b/tools/mcufontencoder/src/export_bwfont.hh new file mode 100644 index 00000000..64b21bc7 --- /dev/null +++ b/tools/mcufontencoder/src/export_bwfont.hh @@ -0,0 +1,16 @@ +// Write out the encoded data in C source code files for mf_bwfont format. + +#pragma once + +#include "datafile.hh" +#include + +namespace mcufont { +namespace bwfont { + +void write_header(std::ostream &out, std::string name, const DataFile &datafile); + +void write_source(std::ostream &out, std::string name, const DataFile &datafile); + +} } + diff --git a/tools/mcufontencoder/src/export_rlefont.cc b/tools/mcufontencoder/src/export_rlefont.cc new file mode 100644 index 00000000..27f91e12 --- /dev/null +++ b/tools/mcufontencoder/src/export_rlefont.cc @@ -0,0 +1,181 @@ +#include "export_rlefont.hh" +#include +#include +#include +#include +#include +#include +#include +#include "exporttools.hh" +#include "ccfixes.hh" + +#define RLEFONT_FORMAT_VERSION 4 + +namespace mcufont { +namespace rlefont { + +// Encode the dictionary entries and the offsets to them. +// Generates tables dictionary_data and dictionary_offsets. +static void encode_dictionary(std::ostream &out, + const std::string &name, + const DataFile &datafile, + const encoded_font_t &encoded) +{ + std::vector offsets; + std::vector data; + for (const encoded_font_t::rlestring_t &r : encoded.rle_dictionary) + { + offsets.push_back(data.size()); + data.insert(data.end(), r.begin(), r.end()); + } + + for (const encoded_font_t::refstring_t &r : encoded.ref_dictionary) + { + offsets.push_back(data.size()); + data.insert(data.end(), r.begin(), r.end()); + } + offsets.push_back(data.size()); + + write_const_table(out, data, "uint8_t", "mf_rlefont_" + name + "_dictionary_data"); + write_const_table(out, offsets, "uint16_t", "mf_rlefont_" + name + "_dictionary_offsets", 4); +} + +// Encode the data tables for a single character range. +// Generates tables glyph_data_i and glyph_offsets_i. +static void encode_character_range(std::ostream &out, + const std::string &name, + const DataFile &datafile, + const encoded_font_t& encoded, + const char_range_t& range, + unsigned range_index) +{ + std::vector offsets; + std::vector data; + std::map already_encoded; + + for (int glyph_index : range.glyph_indices) + { + if (already_encoded.count(glyph_index)) + { + offsets.push_back(already_encoded[glyph_index]); + } + else + { + encoded_font_t::refstring_t r; + int width = 0; + + if (glyph_index >= 0) + { + r = encoded.glyphs[glyph_index]; + width = datafile.GetGlyphEntry(glyph_index).width; + } + + offsets.push_back(data.size()); + already_encoded[glyph_index] = data.size(); + + data.push_back(width); + data.insert(data.end(), r.begin(), r.end()); + } + } + + write_const_table(out, data, "uint8_t", "mf_rlefont_" + name + "_glyph_data_" + std::to_string(range_index)); + write_const_table(out, offsets, "uint16_t", "mf_rlefont_" + name + "_glyph_offsets_" + std::to_string(range_index), 4); +} + +void write_source(std::ostream &out, std::string name, const DataFile &datafile) +{ + name = filename_to_identifier(name); + std::unique_ptr encoded = encode_font(datafile, false); + + out << std::endl; + out << std::endl; + out << "/* Start of automatically generated font definition for " << name << ". */" << std::endl; + out << std::endl; + + out << "#ifndef MF_RLEFONT_INTERNALS" << std::endl; + out << "#define MF_RLEFONT_INTERNALS" << std::endl; + out << "#endif" << std::endl; + out << "#include \"mf_rlefont.h\"" << std::endl; + out << std::endl; + + out << "#ifndef MF_RLEFONT_VERSION_" << RLEFONT_FORMAT_VERSION << "_SUPPORTED" << std::endl; + out << "#error The font file is not compatible with this version of mcufont." << std::endl; + out << "#endif" << std::endl; + out << std::endl; + + // Write out the dictionary entries + encode_dictionary(out, name, datafile, *encoded); + + // Split the characters into ranges + auto get_glyph_size = [&encoded](size_t i) + { + return encoded->glyphs[i].size(); + }; + std::vector ranges = compute_char_ranges(datafile, + get_glyph_size, 65536, 16); + + // Write out glyph data for character ranges + for (size_t i = 0; i < ranges.size(); i++) + { + encode_character_range(out, name, datafile, *encoded, ranges.at(i), i); + } + + // Write out a table describing the character ranges + out << "static const struct mf_rlefont_char_range_s mf_rlefont_" << name << "_char_ranges[] = {" << std::endl; + for (size_t i = 0; i < ranges.size(); i++) + { + out << " {" << ranges.at(i).first_char + << ", " << ranges.at(i).char_count + << ", mf_rlefont_" << name << "_glyph_offsets_" << i + << ", mf_rlefont_" << name << "_glyph_data_" << i << "}," << std::endl; + } + out << "};" << std::endl; + out << std::endl; + + // Pull it all together in the rlefont_s structure. + out << "const struct mf_rlefont_s mf_rlefont_" << name << " = {" << std::endl; + out << " {" << std::endl; + out << " " << "\"" << datafile.GetFontInfo().name << "\"," << std::endl; + out << " " << "\"" << name << "\"," << std::endl; + out << " " << datafile.GetFontInfo().max_width << ", /* width */" << std::endl; + out << " " << datafile.GetFontInfo().max_height << ", /* height */" << std::endl; + out << " " << get_min_x_advance(datafile) << ", /* min x advance */" << std::endl; + out << " " << get_max_x_advance(datafile) << ", /* max x advance */" << std::endl; + out << " " << datafile.GetFontInfo().baseline_x << ", /* baseline x */" << std::endl; + out << " " << datafile.GetFontInfo().baseline_y << ", /* baseline y */" << std::endl; + out << " " << datafile.GetFontInfo().line_height << ", /* line height */" << std::endl; + out << " " << datafile.GetFontInfo().flags << ", /* flags */" << std::endl; + out << " " << select_fallback_char(datafile) << ", /* fallback character */" << std::endl; + out << " " << "&mf_rlefont_character_width," << std::endl; + out << " " << "&mf_rlefont_render_character," << std::endl; + out << " }," << std::endl; + + out << " " << RLEFONT_FORMAT_VERSION << ", /* version */" << std::endl; + out << " " << "mf_rlefont_" << name << "_dictionary_data," << std::endl; + out << " " << "mf_rlefont_" << name << "_dictionary_offsets," << std::endl; + out << " " << encoded->rle_dictionary.size() << ", /* rle dict count */" << std::endl; + out << " " << encoded->ref_dictionary.size() + encoded->rle_dictionary.size() << ", /* total dict count */" << std::endl; + out << " " << ranges.size() << ", /* char range count */" << std::endl; + out << " " << "mf_rlefont_" << name << "_char_ranges," << std::endl; + out << "};" << std::endl; + + // Write the font lookup structure + out << std::endl; + out << "#ifdef MF_INCLUDED_FONTS" << std::endl; + out << "/* List entry for searching fonts by name. */" << std::endl; + out << "static const struct mf_font_list_s mf_rlefont_" << name << "_listentry = {" << std::endl; + out << " MF_INCLUDED_FONTS," << std::endl; + out << " (struct mf_font_s*)&mf_rlefont_" << name << std::endl; + out << "};" << std::endl; + out << "#undef MF_INCLUDED_FONTS" << std::endl; + out << "#define MF_INCLUDED_FONTS (&mf_rlefont_" << name << "_listentry)" << std::endl; + out << "#endif" << std::endl; + + out << std::endl; + out << std::endl; + out << "/* End of automatically generated font definition for " << name << ". */" << std::endl; + out << std::endl; +} + +}} + diff --git a/tools/mcufontencoder/src/export_rlefont.hh b/tools/mcufontencoder/src/export_rlefont.hh new file mode 100644 index 00000000..c01c5644 --- /dev/null +++ b/tools/mcufontencoder/src/export_rlefont.hh @@ -0,0 +1,15 @@ +// Write out the encoded data in C source code files for the mf_rlefont format. + +#pragma once + +#include "datafile.hh" +#include "encode_rlefont.hh" +#include + +namespace mcufont { +namespace rlefont { + +void write_source(std::ostream &out, std::string name, const DataFile &datafile); + +} } + diff --git a/tools/mcufontencoder/src/exporttools.cc b/tools/mcufontencoder/src/exporttools.cc new file mode 100644 index 00000000..b58ee8ec --- /dev/null +++ b/tools/mcufontencoder/src/exporttools.cc @@ -0,0 +1,179 @@ +#include "exporttools.hh" +#include +#include + +namespace mcufont { + + +// Convert a file name to a valid C identifier +std::string filename_to_identifier(std::string name) +{ + // If the name contains path separators (/ or \), take only the last part. + size_t pos = name.find_last_of("/\\"); + if (pos != std::string::npos) + name = name.substr(pos + 1); + + // If the name contains a file extension, strip it. + pos = name.find_first_of("."); + if (pos != std::string::npos) + name = name.substr(0, pos); + + // Replace any special characters with _. + for (pos = 0; pos < name.size(); pos++) + { + if (!isalnum(name.at(pos))) + name.at(pos) = '_'; + } + + return name; +} + +// Write a vector of integers as line-wrapped hex/integer data for initializing const array. +void wordwrap_vector(std::ostream &out, const std::vector &data, + const std::string &prefix, size_t width) +{ + int values_per_column = (width <= 2) ? 16 : 8; + + std::ios::fmtflags flags(out.flags()); + out << prefix; + out << std::hex << std::setfill('0'); + for (size_t i = 0; i < data.size(); i++) + { + if (i % values_per_column == 0 && i != 0) + out << std::endl << prefix; + + out << "0x" << std::setw(width) << (int)data.at(i) << ", "; + } + out.flags(flags); +} + +// Write a vector of integers as a C constant array of given datatype. + void write_const_table(std::ostream &out, const std::vector &data, + const std::string &datatype, const std::string &tablename, + size_t width) +{ + out << "static const " << datatype << " " << tablename; + out << "[" << data.size() << "] = {" << std::endl; + wordwrap_vector(out, data, " ", width); + out << std::endl << "};" << std::endl; + out << std::endl; +} + +int get_min_x_advance(const DataFile &datafile) +{ + int min = datafile.GetGlyphEntry(0).width; + + for (const DataFile::glyphentry_t &g : datafile.GetGlyphTable()) + { + if (min > g.width) + min = g.width; + } + + return min; +} + +int get_max_x_advance(const DataFile &datafile) +{ + int max = 0; + + for (const DataFile::glyphentry_t &g : datafile.GetGlyphTable()) + { + if (max < g.width) + max = g.width; + } + + return max; +} + +// Select the character to use as a fallback. +int select_fallback_char(const DataFile &datafile) +{ + std::set chars; + + size_t i = 0; + for (const DataFile::glyphentry_t &g: datafile.GetGlyphTable()) + { + for (size_t c: g.chars) + { + chars.insert(c); + } + i++; + } + + if (chars.count(0xFFFD)) + return 0xFFFD; // Unicode replacement character + + if (chars.count(0)) + return 0; // Used by many BDF fonts as replacement char + + if (chars.count('?')) + return '?'; + + return ' '; +} + +// Decide how to best divide the characters in the font into ranges. +// Limitations are: +// - Gaps longer than minimum_gap should result in separate ranges. +// - Each range can have encoded data size of at most maximum_size. +std::vector compute_char_ranges(const DataFile &datafile, + std::function get_encoded_glyph_size, + size_t maximum_size, + size_t minimum_gap) +{ + std::vector result; + std::map char_to_glyph = datafile.GetCharToGlyphMap(); + std::vector chars; + + // Get list of all characters in numeric order. + for (auto iter : char_to_glyph) + chars.push_back(iter.first); + + // Pick out ranges until we have processed all characters + size_t i = 0; + while (i < chars.size()) + { + char_range_t range; + range.first_char = chars.at(i); + + // Find the point where there is a gap larger than minimum_gap. + i++; + while (i < chars.size() && chars.at(i) - chars.at(i - 1) < minimum_gap) + i++; + + uint16_t last_char = chars.at(i - 1); + + // Then store the indices of glyphs for each character + size_t data_length = 0; + for (size_t j = range.first_char; j <= last_char; j++) + { + if (char_to_glyph.count(j) == 0) + { + // Missing character + range.glyph_indices.push_back(-1); + continue; + } + + int glyph_index = char_to_glyph[j]; + + // Monitor the amount of the data in the range and split it + // if it grows too large. + data_length += get_encoded_glyph_size(glyph_index); + if (data_length > maximum_size) + { + last_char = j - 1; + break; + } + + range.glyph_indices.push_back(glyph_index); + } + + range.char_count = last_char - range.first_char + 1; + result.push_back(range); + } + + return result; +} + + +} diff --git a/tools/mcufontencoder/src/exporttools.hh b/tools/mcufontencoder/src/exporttools.hh new file mode 100644 index 00000000..383d3b11 --- /dev/null +++ b/tools/mcufontencoder/src/exporttools.hh @@ -0,0 +1,52 @@ +// Utility functions for exporting to C source code files. + +#pragma once +#include +#include +#include +#include +#include "datafile.hh" + +namespace mcufont { + +// Convert a file name to a valid C identifier +std::string filename_to_identifier(std::string name); + +// Write a vector of integers as line-wrapped hex/integer data for initializing const array. +void wordwrap_vector(std::ostream &out, const std::vector &data, + const std::string &prefix, size_t width = 2); + +// Write a vector of integers as a C constant array of given datatype. +void write_const_table(std::ostream &out, const std::vector &data, + const std::string &datatype, const std::string &tablename, + size_t width = 2); + +// Get minimum tracking width of font +int get_min_x_advance(const DataFile &datafile); + +// Get maximum tracking width of font +int get_max_x_advance(const DataFile &datafile); + +// Select the character to use as a fallback. +int select_fallback_char(const DataFile &datafile); + +// Structure to represent one consecutive range of characters. +struct char_range_t +{ + uint16_t first_char; + uint16_t char_count; + std::vector glyph_indices; + + char_range_t(): first_char(0), char_count(0) {} +}; + +// Decide how to best divide the characters in the font into ranges. +// Limitations are: +// - Gaps longer than minimum_gap should result in separate ranges. +// - Each range can have encoded data size of at most maximum_size. +std::vector compute_char_ranges(const DataFile &datafile, + std::function get_encoded_glyph_size, + size_t maximum_size, + size_t minimum_gap); + +} \ No newline at end of file diff --git a/tools/mcufontencoder/src/freetype_import.cc b/tools/mcufontencoder/src/freetype_import.cc new file mode 100644 index 00000000..27a0734d --- /dev/null +++ b/tools/mcufontencoder/src/freetype_import.cc @@ -0,0 +1,177 @@ +#include "freetype_import.hh" +#include "importtools.hh" +#include +#include +#include +#include +#include "ccfixes.hh" + +#include +#include FT_FREETYPE_H + +#undef __FTERRORS_H__ +#define FT_ERRORDEF( e, v, s ) std::make_pair( e, s ), +#define FT_ERROR_START_LIST static const std::map ft_errors { +#define FT_ERROR_END_LIST }; +#include FT_ERRORS_H + +namespace mcufont { + +static void checkFT(FT_Error error) +{ + if (error != 0) + { + if (ft_errors.count(error)) + throw std::runtime_error("libfreetype error " + + std::to_string(error) + ": " + ft_errors.at(error)); + else + throw std::runtime_error("unknown libfreetype error " + + std::to_string(error)); + } +} + +// Automatically allocated & freed wrapper for FT_Library +class _FT_Library +{ +public: + _FT_Library() { checkFT(FT_Init_FreeType(&m_lib)); } + ~_FT_Library() { checkFT(FT_Done_FreeType(m_lib)); } + operator FT_Library() { return m_lib; } + +private: + FT_Library m_lib; +}; + +// Automatically allocated & freed wrapper for FT_Face +class _FT_Face +{ +public: + _FT_Face(FT_Library lib, const std::vector &data) + { + checkFT(FT_New_Memory_Face(lib, (const unsigned char *)&data[0], + data.size(), 0, &m_face)); + } + ~_FT_Face() { checkFT(FT_Done_Face(m_face)); } + operator FT_Face() { return m_face; } + FT_Face operator->() { return m_face; } + +private: + FT_Face m_face; +}; + +// Read all the data from a file into a memory buffer. +static void readfile(std::istream &file, std::vector &data) +{ + while (file.good()) + { + const size_t blocksize = 4096; + size_t oldsize = data.size(); + data.resize(oldsize + blocksize); + file.read(&data[oldsize], blocksize); + data.resize(oldsize + file.gcount()); + } +} + +std::unique_ptr LoadFreetype(std::istream &file, int size, bool bw) +{ + std::vector data; + readfile(file, data); + + _FT_Library lib; + _FT_Face face(lib, data); + + checkFT(FT_Set_Pixel_Sizes(face, size, size)); + + DataFile::fontinfo_t fontinfo = {}; + std::vector glyphtable; + std::vector dictionary; + + // Convert size to pixels and round to nearest. + int u_per_em = face->units_per_EM; + auto topx = [size, u_per_em](int s) { return (s * size + u_per_em / 2) / u_per_em; }; + + fontinfo.name = std::string(face->family_name) + " " + + std::string(face->style_name) + " " + + std::to_string(size); + + // Reserve 4 pixels on each side for antialiasing + hinting. + // They will be cropped off later. + fontinfo.max_width = topx(face->bbox.xMax - face->bbox.xMin) + 8; + fontinfo.max_height = topx(face->bbox.yMax - face->bbox.yMin) + 8; + fontinfo.baseline_x = topx(-face->bbox.xMin) + 4; + fontinfo.baseline_y = topx(face->bbox.yMax) + 4; + fontinfo.line_height = topx(face->height); + + FT_Int32 loadmode = FT_LOAD_TARGET_NORMAL | FT_LOAD_RENDER; + + if (bw) + loadmode = FT_LOAD_TARGET_MONO | FT_LOAD_MONOCHROME | FT_LOAD_RENDER; + + FT_ULong charcode; + FT_UInt gindex; + charcode = FT_Get_First_Char(face, &gindex); + while (gindex) + { + try + { + checkFT(FT_Load_Glyph(face, gindex, loadmode)); + } + catch (std::runtime_error &e) + { + std::cerr << "Skipping glyph " << gindex << ": " << e.what() << std::endl; + charcode = FT_Get_Next_Char(face, charcode, &gindex); + } + + DataFile::glyphentry_t glyph; + glyph.width = (face->glyph->advance.x + 32) / 64; + glyph.chars.push_back(charcode); + glyph.data.resize(fontinfo.max_width * fontinfo.max_height); + + int w = face->glyph->bitmap.width; + int dw = fontinfo.max_width; + int dx = fontinfo.baseline_x + face->glyph->bitmap_left; + int dy = fontinfo.baseline_y - face->glyph->bitmap_top; + + /* Some combining diacritics seem to exceed the bounding box. + * We don't support them all that well anyway, so just move + * them inside the box in order not to crash.. */ + if (dy < 0) + dy = 0; + if (dy + face->glyph->bitmap.rows > fontinfo.max_height) + dy = fontinfo.max_height - face->glyph->bitmap.rows; + + size_t s = face->glyph->bitmap.pitch; + for (int y = 0; y < face->glyph->bitmap.rows; y++) + { + for (int x = 0; x < face->glyph->bitmap.width; x++) + { + size_t index = (y + dy) * dw + x + dx; + + if (face->glyph->bitmap.pixel_mode == FT_PIXEL_MODE_MONO) + { + uint8_t byte = face->glyph->bitmap.buffer[s * y + x / 8]; + byte <<= x % 8; + glyph.data.at(index) = (byte & 0x80) ? 15 : 0; + } + else + { + glyph.data.at(index) = + (face->glyph->bitmap.buffer[w * y + x] + 8) / 17; + } + } + } + glyphtable.push_back(glyph); + + charcode = FT_Get_Next_Char(face, charcode, &gindex); + } + + eliminate_duplicates(glyphtable); + crop_glyphs(glyphtable, fontinfo); + detect_flags(glyphtable, fontinfo); + + std::unique_ptr result(new DataFile( + dictionary, glyphtable, fontinfo)); + return result; +} + +} diff --git a/tools/mcufontencoder/src/freetype_import.hh b/tools/mcufontencoder/src/freetype_import.hh new file mode 100644 index 00000000..eacb8b91 --- /dev/null +++ b/tools/mcufontencoder/src/freetype_import.hh @@ -0,0 +1,10 @@ +// Function for importing any font supported by libfreetype. + +#pragma once +#include "datafile.hh" + +namespace mcufont { + +std::unique_ptr LoadFreetype(std::istream &file, int size, bool bw); + +} diff --git a/tools/mcufontencoder/src/importtools.cc b/tools/mcufontencoder/src/importtools.cc new file mode 100644 index 00000000..c219c207 --- /dev/null +++ b/tools/mcufontencoder/src/importtools.cc @@ -0,0 +1,134 @@ +#include "importtools.hh" +#include + +namespace mcufont { + +void eliminate_duplicates(std::vector &glyphtable) +{ + for (size_t i = 0; i + 1 < glyphtable.size(); i++) + { + for (size_t j = i + 1; j < glyphtable.size(); j++) + { + if (glyphtable.at(i).data == glyphtable.at(j).data && + glyphtable.at(i).width == glyphtable.at(j).width) + { + for (int c : glyphtable.at(j).chars) + glyphtable.at(i).chars.push_back(c); + + glyphtable.erase(glyphtable.begin() + j); + j--; + } + } + } +} + +struct bbox_t +{ + int left; + int top; + int right; + int bottom; + + bbox_t() + { + left = std::numeric_limits::max(); + top = std::numeric_limits::max(); + right = std::numeric_limits::min(); + bottom = std::numeric_limits::min(); + } + + void update(int x, int y) + { + if (x < left) left = x; + if (x > right) right = x; + if (y < top) top = y; + if (y > bottom) bottom = y; + } +}; + +void crop_glyphs(std::vector &glyphtable, + DataFile::fontinfo_t &fontinfo) +{ + // Find out the maximum bounding box + bbox_t bbox; + for (DataFile::glyphentry_t &glyph : glyphtable) + { + for (int y = 0; y < fontinfo.max_height; y++) + { + for (int x = 0; x < fontinfo.max_width; x++) + { + if (glyph.data.at(y * fontinfo.max_width + x)) + bbox.update(x, y); + } + } + } + + // Crop the glyphs to that + size_t old_w = fontinfo.max_width; + size_t new_w = bbox.right - bbox.left + 1; + size_t new_h = bbox.bottom - bbox.top + 1; + for (DataFile::glyphentry_t &glyph : glyphtable) + { + DataFile::pixels_t old = glyph.data; + glyph.data.clear(); + + for (size_t y = 0; y < new_h; y++) + { + for (size_t x = 0; x < new_w; x++) + { + size_t old_x = bbox.left + x; + size_t old_y = bbox.top + y; + size_t old_pos = old_w * old_y + old_x; + glyph.data.push_back(old.at(old_pos)); + } + } + } + + fontinfo.max_width = new_w; + fontinfo.max_height = new_h; + fontinfo.baseline_x -= bbox.left; + fontinfo.baseline_y -= bbox.top; +} + +void detect_flags(const std::vector &glyphtable, + DataFile::fontinfo_t &fontinfo) +{ + if (!glyphtable.size()) + return; + + // Check if all glyphs have equal width + int width = glyphtable[0].width; + bool is_monospace = true; + for (const DataFile::glyphentry_t &g : glyphtable) + { + if (g.width != width) + { + is_monospace = false; + break; + } + } + + if (is_monospace) + fontinfo.flags |= DataFile::FLAG_MONOSPACE; + + // Check if all glyphs contain only 0 or 15 alpha + bool is_bw = true; + for (const DataFile::glyphentry_t &g : glyphtable) + { + for (uint8_t pixel : g.data) + { + if (pixel != 0 && pixel != 15) + { + is_bw = false; + break; + } + } + if (!is_bw) break; + } + + if (is_bw) + fontinfo.flags |= DataFile::FLAG_BW; +} + + +} diff --git a/tools/mcufontencoder/src/importtools.hh b/tools/mcufontencoder/src/importtools.hh new file mode 100644 index 00000000..20371f22 --- /dev/null +++ b/tools/mcufontencoder/src/importtools.hh @@ -0,0 +1,20 @@ +// Utility functions for processing imported font files. + +#pragma once +#include "datafile.hh" + +namespace mcufont { + +// Find and eliminate any duplicate glyphs by appending their char vectors. +void eliminate_duplicates(std::vector &glyphtable); + +// Calculate the maximum bounding box of the glyphs and crop them to that. +// Adjust fontinfo accordingly. +void crop_glyphs(std::vector &glyphtable, + DataFile::fontinfo_t &fontinfo); + +// Fill in the flags (BW, monospace) automatically. +void detect_flags(const std::vector &glyphtable, + DataFile::fontinfo_t &fontinfo); + +} diff --git a/tools/mcufontencoder/src/libfreetype.dll.a b/tools/mcufontencoder/src/libfreetype.dll.a new file mode 100644 index 00000000..4d040cb1 Binary files /dev/null and b/tools/mcufontencoder/src/libfreetype.dll.a differ diff --git a/tools/mcufontencoder/src/main.cc b/tools/mcufontencoder/src/main.cc new file mode 100644 index 00000000..19c9c6f7 --- /dev/null +++ b/tools/mcufontencoder/src/main.cc @@ -0,0 +1,480 @@ +#include "datafile.hh" +#include "importtools.hh" +#include "bdf_import.hh" +#include "freetype_import.hh" +#include "export_rlefont.hh" +#include "encode_rlefont.hh" +#include "optimize_rlefont.hh" +#include "export_bwfont.hh" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ccfixes.hh" + +using namespace mcufont; + +static std::string strip_extension(std::string filename) +{ + size_t pos = filename.find_last_of('.'); + + if (pos == std::string::npos) + { + return filename; + } + else + { + return filename.substr(0, pos); + } +} + +static std::unique_ptr load_dat(std::string src) +{ + std::ifstream infile(src); + + if (!infile.good()) + { + std::cerr << "Could not open " << src << std::endl; + return nullptr; + } + + std::unique_ptr f = DataFile::Load(infile); + if (!f) + { + std::cerr << "Invalid format for .dat file: " << src << std::endl; + return nullptr; + } + + return f; +} + +static bool save_dat(std::string dest, DataFile *f) +{ + std::ofstream outfile(dest); + + if (!outfile.good()) + { + std::cerr << "Could not open " << dest << std::endl; + return false; + } + + f->Save(outfile); + + if (!outfile.good()) + { + std::cerr << "Could not write to " << dest << std::endl; + return false; + } + + return true; +} + +enum status_t +{ + STATUS_OK = 0, // All good + STATUS_INVALID = 1, // Invalid command or args + STATUS_ERROR = 2 // Error when executing command +}; + +static status_t cmd_import_ttf(const std::vector &args) +{ + if (args.size() != 3 && args.size() != 4) + return STATUS_INVALID; + + std::string src = args.at(1); + int size = std::stoi(args.at(2)); + bool bw = (args.size() == 4 && args.at(3) == "bw"); + std::string dest = strip_extension(src) + std::to_string(size) + (bw ? "bw" : "") + ".dat"; + std::ifstream infile(src); + + if (!infile.good()) + { + std::cerr << "Could not open " << src << std::endl; + return STATUS_ERROR; + } + + std::cout << "Importing " << src << " to " << dest << std::endl; + + std::unique_ptr f = LoadFreetype(infile, size, bw); + + mcufont::rlefont::init_dictionary(*f); + + if (!save_dat(dest, f.get())) + return STATUS_ERROR; + + std::cout << "Done: " << f->GetGlyphCount() << " unique glyphs." << std::endl; + return STATUS_OK; +} + +static status_t cmd_import_bdf(const std::vector &args) +{ + if (args.size() != 2) + return STATUS_INVALID; + + std::string src = args.at(1); + std::string dest = strip_extension(args.at(1)) + ".dat"; + std::ifstream infile(src); + + if (!infile.good()) + { + std::cerr << "Could not open " << src << std::endl; + return STATUS_ERROR; + } + + std::cout << "Importing " << src << " to " << dest << std::endl; + + std::unique_ptr f = LoadBDF(infile); + + mcufont::rlefont::init_dictionary(*f); + + if (!save_dat(dest, f.get())) + return STATUS_ERROR; + + std::cout << "Done: " << f->GetGlyphCount() << " unique glyphs." << std::endl; + return STATUS_OK; +} + +static status_t cmd_filter(const std::vector &args) +{ + if (args.size() < 3) + return STATUS_INVALID; + + std::set allowed; + + // Parse arguments + for (size_t i = 2; i < args.size(); i++) + { + std::string s = args.at(i); + size_t pos = s.find('-'); + if (pos == std::string::npos) + { + // Single char + allowed.insert(std::stoi(s, nullptr, 0)); + } + else + { + // Range + int start = std::stoi(s.substr(0, pos), nullptr, 0); + int end = std::stoi(s.substr(pos + 1), nullptr, 0); + + for (int j = start; j <= end; j++) + { + allowed.insert(j); + } + } + } + + std::string src = args.at(1); + std::unique_ptr f = load_dat(src); + if (!f) + return STATUS_ERROR; + + std::cout << "Font originally had " << f->GetGlyphCount() << " glyphs." << std::endl; + + // Filter the glyphs + std::vector newglyphs; + for (size_t i = 0; i < f->GetGlyphCount(); i++) + { + DataFile::glyphentry_t g = f->GetGlyphEntry(i); + + for (size_t j = 0; j < g.chars.size(); j++) + { + if (!allowed.count(g.chars.at(j))) + { + g.chars.erase(g.chars.begin() + j); + j--; + } + } + + if (g.chars.size()) + { + newglyphs.push_back(g); + } + } + + DataFile::fontinfo_t fontinfo = f->GetFontInfo(); + crop_glyphs(newglyphs, fontinfo); + detect_flags(newglyphs, fontinfo); + + f.reset(new DataFile(f->GetDictionary(), newglyphs, fontinfo)); + std::cout << "After filtering, " << f->GetGlyphCount() << " glyphs remain." << std::endl; + + if (!save_dat(src, f.get())) + return STATUS_ERROR; + + return STATUS_OK; +} + +static status_t cmd_show_glyph(const std::vector &args) +{ + if (args.size() != 3) + return STATUS_INVALID; + + std::string src = args.at(1); + std::unique_ptr f = load_dat(src); + + if (!f) + return STATUS_ERROR; + + size_t index = 0; + if (args.at(2) == "largest") + { + std::unique_ptr e = + mcufont::rlefont::encode_font(*f, false); + size_t maxlen = 0; + size_t i = 0; + for (mcufont::rlefont::encoded_font_t::refstring_t g : e->glyphs) + { + if (g.size() > maxlen) + { + maxlen = g.size(); + index = i; + } + i++; + } + + std::cout << "Index " << index << ", length " << maxlen << std::endl; + } + else + { + index = strtol(args.at(2).c_str(), nullptr, 0); + } + + if (index < 0 || index >= f->GetGlyphCount()) + { + std::cerr << "No such glyph " << index << std::endl; + return STATUS_ERROR; + } + + std::cout << "Width: " << f->GetGlyphEntry(index).width << std::endl; + std::cout << "Chars: "; + for (int c: f->GetGlyphEntry(index).chars) std::cout << c << " "; + std::cout << std::endl; + + std::cout << f->GlyphToText(index); + return STATUS_OK; +} + +static status_t cmd_rlefont_export(const std::vector &args) +{ + if (args.size() != 2 && args.size() != 3) + return STATUS_INVALID; + + std::string src = args.at(1); + std::string dst = (args.size() == 2) ? strip_extension(src) + ".c" : args.at(2); + std::unique_ptr f = load_dat(src); + + if (!f) + return STATUS_ERROR; + + { + std::ofstream source(dst); + mcufont::rlefont::write_source(source, dst, *f); + std::cout << "Wrote " << dst << std::endl; + } + + return STATUS_OK; +} + +static status_t cmd_rlefont_size(const std::vector &args) +{ + if (args.size() != 2) + return STATUS_INVALID; + + std::string src = args.at(1); + std::unique_ptr f = load_dat(src); + + if (!f) + return STATUS_ERROR; + + size_t size = mcufont::rlefont::get_encoded_size(*f); + + std::cout << "Glyph count: " << f->GetGlyphCount() << std::endl; + std::cout << "Glyph bbox: " << f->GetFontInfo().max_width << "x" + << f->GetFontInfo().max_height << " pixels" << std::endl; + std::cout << "Uncompressed size: " << f->GetGlyphCount() * + f->GetFontInfo().max_width * f->GetFontInfo().max_height / 2 + << " bytes" << std::endl; + std::cout << "Compressed size: " << size << " bytes" << std::endl; + std::cout << "Bytes per glyph: " << size / f->GetGlyphCount() << std::endl; + return STATUS_OK; +} + +static status_t cmd_rlefont_optimize(const std::vector &args) +{ + if (args.size() != 2 && args.size() != 3) + return STATUS_INVALID; + + std::string src = args.at(1); + std::unique_ptr f = load_dat(src); + + if (!f) + return STATUS_ERROR; + + size_t oldsize = mcufont::rlefont::get_encoded_size(*f); + + std::cout << "Original size is " << oldsize << " bytes" << std::endl; + std::cout << "Press ctrl-C at any time to stop." << std::endl; + std::cout << "Results are saved automatically after each iteration." << std::endl; + + int limit = 100; + if (args.size() == 3) + { + limit = std::stoi(args.at(2)); + } + + if (limit > 0) + std::cout << "Limit is " << limit << " iterations" << std::endl; + + int i = 0; + time_t oldtime = time(NULL); + while (!limit || i < limit) + { + mcufont::rlefont::optimize(*f); + + size_t newsize = mcufont::rlefont::get_encoded_size(*f); + time_t newtime = time(NULL); + + int bytes_per_min = (oldsize - newsize) * 60 / (newtime - oldtime + 1); + + i++; + std::cout << "iteration " << i << ", size " << newsize + << " bytes, speed " << bytes_per_min << " B/min" + << std::endl; + + { + if (!save_dat(src, f.get())) + return STATUS_ERROR; + } + } + + return STATUS_OK; +} + +static status_t cmd_rlefont_show_encoded(const std::vector &args) +{ + if (args.size() != 2) + return STATUS_INVALID; + + std::string src = args.at(1); + std::unique_ptr f = load_dat(src); + + if (!f) + return STATUS_ERROR; + + std::unique_ptr e = + mcufont::rlefont::encode_font(*f, false); + + int i = 0; + for (mcufont::rlefont::encoded_font_t::rlestring_t d : e->rle_dictionary) + { + std::cout << "Dict RLE " << 24 + i++ << ": "; + for (uint8_t v : d) + std::cout << std::setfill('0') << std::setw(2) << std::hex << (int)v << " "; + std::cout << std::endl; + } + + for (mcufont::rlefont::encoded_font_t::refstring_t d : e->ref_dictionary) + { + std::cout << "Dict Ref " << 24 + i++ << ": "; + for (uint8_t v : d) + std::cout << std::setfill('0') << std::setw(2) << std::hex << (int)v << " "; + std::cout << std::endl; + } + + i = 0; + for (mcufont::rlefont::encoded_font_t::refstring_t g : e->glyphs) + { + std::cout << "Glyph " << i++ << ": "; + for (uint8_t v : g) + std::cout << std::setfill('0') << std::setw(2) << std::hex << (int)v << " "; + std::cout << std::endl; + } + + return STATUS_OK; +} + +static status_t cmd_bwfont_export(const std::vector &args) +{ + if (args.size() != 2 && args.size() != 3) + return STATUS_INVALID; + + std::string src = args.at(1); + std::string dst = (args.size() == 2) ? strip_extension(src) + ".c" : args.at(2); + std::unique_ptr f = load_dat(src); + + if (!f) + return STATUS_ERROR; + + if (!(f->GetFontInfo().flags & DataFile::FLAG_BW)) + { + std::cout << "Warning: font is not black and white" << std::endl; + } + + { + std::ofstream source(dst); + mcufont::bwfont::write_source(source, dst, *f); + std::cout << "Wrote " << dst << std::endl; + } + + return STATUS_OK; +} + + +static const char *usage_msg = + "Usage: mcufont [options] ...\n" + "Commands for importing:\n" + " import_ttf [bw] Import a .ttf font into a data file.\n" + " import_bdf Import a .bdf font into a data file.\n" + "\n" + "Commands for inspecting and editing data files:\n" + " filter ... Remove everything except specified characters.\n" + " show_glyph Show the glyph at index.\n" + "\n" + "Commands specific to rlefont format:\n" + " rlefont_size Check the encoded size of the data file.\n" + " rlefont_optimize Perform an optimization pass on the data file.\n" + " rlefont_export [outfile] Export to .c source code.\n" + " rlefont_show_encoded Show the encoded data for debugging.\n" + "\n" + "Commands specific to bwfont format:\n" + " bwfont_export [outfile Export to .c source code.\n" + ""; + +typedef status_t (*cmd_t)(const std::vector &args); +static const std::map command_list { + {"import_ttf", cmd_import_ttf}, + {"import_bdf", cmd_import_bdf}, + {"filter", cmd_filter}, + {"show_glyph", cmd_show_glyph}, + {"rlefont_size", cmd_rlefont_size}, + {"rlefont_optimize", cmd_rlefont_optimize}, + {"rlefont_export", cmd_rlefont_export}, + {"rlefont_show_encoded", cmd_rlefont_show_encoded}, + {"bwfont_export", cmd_bwfont_export}, +}; + +int main(int argc, char **argv) +{ + std::vector args; + for (int i = 1; i < argc; i++) + args.push_back(argv[i]); + + status_t status = STATUS_INVALID; + if (args.size() >= 1 && command_list.count(args.at(0))) + { + status = command_list.find(args.at(0))->second(args); + } + + if (status == STATUS_INVALID) + { + std::cout << usage_msg << std::endl; + } + + return status; +} diff --git a/tools/mcufontencoder/src/optimize_rlefont.cc b/tools/mcufontencoder/src/optimize_rlefont.cc new file mode 100644 index 00000000..20c340db --- /dev/null +++ b/tools/mcufontencoder/src/optimize_rlefont.cc @@ -0,0 +1,417 @@ +#include "optimize_rlefont.hh" +#include "encode_rlefont.hh" +#include +#include +#include +#include +#include +#include "ccfixes.hh" + +namespace mcufont { +namespace rlefont { + +typedef std::mt19937 rnd_t; + +// Select a random substring among all the glyphs in the datafile. +std::unique_ptr random_substring(const DataFile &datafile, rnd_t &rnd) +{ + std::uniform_int_distribution dist1(0, datafile.GetGlyphCount() - 1); + size_t index = dist1(rnd); + + const DataFile::pixels_t &pixels = datafile.GetGlyphEntry(index).data; + + std::uniform_int_distribution dist2(2, pixels.size()); + size_t length = dist2(rnd); + + std::uniform_int_distribution dist3(0, pixels.size() - length); + size_t start = dist3(rnd); + + std::unique_ptr result; + result.reset(new DataFile::pixels_t(pixels.begin() + start, + pixels.begin() + start + length)); + return result; +} + +// Try to replace the worst dictionary entry with a better one. +void optimize_worst(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose) +{ + std::uniform_int_distribution dist(0, 1); + + DataFile trial = datafile; + size_t worst = trial.GetLowScoreIndex(); + DataFile::dictentry_t d = trial.GetDictionaryEntry(worst); + d.replacement = *random_substring(datafile, rnd); + d.ref_encode = dist(rnd); + trial.SetDictionaryEntry(worst, d); + + size_t newsize = get_encoded_size(trial); + + if (newsize < size) + { + d.score = size - newsize; + datafile.SetDictionaryEntry(worst, d); + size = newsize; + + if (verbose) + std::cout << "optimize_worst: replaced " << worst + << " score " << d.score << std::endl; + } +} + +// Try to replace random dictionary entry with another one. +void optimize_any(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose) +{ + DataFile trial = datafile; + std::uniform_int_distribution dist(0, DataFile::dictionarysize - 1); + size_t index = dist(rnd); + DataFile::dictentry_t d = trial.GetDictionaryEntry(index); + d.replacement = *random_substring(datafile, rnd); + trial.SetDictionaryEntry(index, d); + + size_t newsize = get_encoded_size(trial); + + if (newsize < size) + { + d.score = size - newsize; + datafile.SetDictionaryEntry(index, d); + size = newsize; + + if (verbose) + std::cout << "optimize_any: replaced " << index + << " score " << d.score << std::endl; + } +} + +// Try to append or prepend random dictionary entry. +void optimize_expand(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose, bool binary_only) +{ + DataFile trial = datafile; + std::uniform_int_distribution dist1(0, DataFile::dictionarysize - 1); + size_t index = dist1(rnd); + DataFile::dictentry_t d = trial.GetDictionaryEntry(index); + + std::uniform_int_distribution dist3(1, 3); + size_t count = dist3(rnd); + + for (size_t i = 0; i < count; i++) + { + std::uniform_int_distribution booldist(0, 1); + std::uniform_int_distribution pixeldist(0, 15); + uint8_t pixel; + + if (binary_only) + { + pixel = booldist(rnd) ? 15 : 0; + } + else + { + pixel = pixeldist(rnd); + } + + bool prepend = booldist(rnd); + + if (prepend) + { + d.replacement.insert(d.replacement.begin(), pixel); + } + else + { + d.replacement.push_back(pixel); + } + } + + trial.SetDictionaryEntry(index, d); + + size_t newsize = get_encoded_size(trial); + + if (newsize < size) + { + d.score = size - newsize; + datafile.SetDictionaryEntry(index, d); + size = newsize; + + if (verbose) + std::cout << "optimize_expand: expanded " << index + << " by " << count << " pixels, score " << d.score << std::endl; + } +} + +// Try to trim random dictionary entry. +void optimize_trim(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose) +{ + DataFile trial = datafile; + std::uniform_int_distribution dist1(0, DataFile::dictionarysize - 1); + size_t index = dist1(rnd); + DataFile::dictentry_t d = trial.GetDictionaryEntry(index); + + if (d.replacement.size() <= 2) return; + + std::uniform_int_distribution dist2(0, std::min((int)d.replacement.size() / 2, 5)); + size_t start = dist2(rnd); + size_t end = dist2(rnd); + + if (start) + { + d.replacement.erase(d.replacement.begin(), d.replacement.begin() + start); + } + + if (end) + { + d.replacement.erase(d.replacement.end() - end, d.replacement.end() - 1); + } + + trial.SetDictionaryEntry(index, d); + + size_t newsize = get_encoded_size(trial); + + if (newsize < size) + { + d.score = size - newsize; + datafile.SetDictionaryEntry(index, d); + size = newsize; + + if (verbose) + std::cout << "optimize_trim: trimmed " << index + << " by " << start << " pixels from start and " + << end << " pixels from end, score " << d.score << std::endl; + } +} + +// Switch random dictionary entry to use ref encoding or back to rle. +void optimize_refdict(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose) +{ + DataFile trial = datafile; + std::uniform_int_distribution dist1(0, DataFile::dictionarysize - 1); + size_t index = dist1(rnd); + DataFile::dictentry_t d = trial.GetDictionaryEntry(index); + + d.ref_encode = !d.ref_encode; + + trial.SetDictionaryEntry(index, d); + + size_t newsize = get_encoded_size(trial); + + if (newsize < size) + { + d.score = size - newsize; + datafile.SetDictionaryEntry(index, d); + size = newsize; + + if (verbose) + std::cout << "optimize_refdict: switched " << index + << " to " << (d.ref_encode ? "ref" : "RLE") + << ", score " << d.score << std::endl; + } +} + +// Combine two random dictionary entries. +void optimize_combine(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose) +{ + DataFile trial = datafile; + std::uniform_int_distribution dist1(0, DataFile::dictionarysize - 1); + size_t worst = datafile.GetLowScoreIndex(); + size_t index1 = dist1(rnd); + size_t index2 = dist1(rnd); + + const DataFile::pixels_t &part1 = datafile.GetDictionaryEntry(index1).replacement; + const DataFile::pixels_t &part2 = datafile.GetDictionaryEntry(index2).replacement; + + DataFile::dictentry_t d; + d.replacement = part1; + d.replacement.insert(d.replacement.end(), part2.begin(), part2.end()); + d.ref_encode = true; + trial.SetDictionaryEntry(worst, d); + + size_t newsize = get_encoded_size(trial); + + if (newsize < size) + { + d.score = size - newsize; + datafile.SetDictionaryEntry(worst, d); + size = newsize; + + if (verbose) + std::cout << "optimize_combine: combined " << index1 + << " and " << index2 << " to replace " << worst + << ", score " << d.score << std::endl; + } +} + +// Pick a random part of an encoded glyph and encode it as a ref dict. +void optimize_encpart(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose) +{ + std::unique_ptr e = encode_font(datafile); + + // Pick a random encoded glyph + std::uniform_int_distribution dist1(0, datafile.GetGlyphCount() - 1); + size_t index = dist1(rnd); + const encoded_font_t::refstring_t &refstr = e->glyphs.at(index); + + if (refstr.size() < 2) + return; + + // Pick a random part of it + std::uniform_int_distribution dist2(2, refstr.size()); + size_t length = dist2(rnd); + std::uniform_int_distribution dist3(0, refstr.size() - length); + size_t start = dist3(rnd); + + // Decode that part + encoded_font_t::refstring_t substr(refstr.begin() + start, + refstr.begin() + start + length); + std::unique_ptr decoded = + decode_glyph(*e, substr, datafile.GetFontInfo()); + + // Add that as a new dictionary entry + DataFile trial = datafile; + size_t worst = trial.GetLowScoreIndex(); + DataFile::dictentry_t d = trial.GetDictionaryEntry(worst); + d.replacement = *decoded; + d.ref_encode = true; + trial.SetDictionaryEntry(worst, d); + + size_t newsize = get_encoded_size(trial); + + if (newsize < size) + { + d.score = size - newsize; + datafile.SetDictionaryEntry(worst, d); + size = newsize; + + if (verbose) + std::cout << "optimize_encpart: replaced " << worst + << " score " << d.score << std::endl; + } +} + +// Execute all the optimization algorithms once. +void optimize_pass(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose) +{ + optimize_worst(datafile, size, rnd, verbose); + optimize_any(datafile, size, rnd, verbose); + optimize_expand(datafile, size, rnd, verbose, false); + optimize_expand(datafile, size, rnd, verbose, true); + optimize_trim(datafile, size, rnd, verbose); + optimize_refdict(datafile, size, rnd, verbose); + optimize_combine(datafile, size, rnd, verbose); + optimize_encpart(datafile, size, rnd, verbose); +} + +// Execute multiple passes in parallel and take the one with the best result. +// The amount of parallelism is hardcoded in order to retain deterministic +// behaviour. +void optimize_parallel(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose, int num_threads = 4) +{ + std::vector datafiles; + std::vector sizes; + std::vector rnds; + std::vector > threads; + + for (int i = 0; i < num_threads; i++) + { + datafiles.emplace_back(datafile); + sizes.emplace_back(size); + rnds.emplace_back(rnd()); + } + + for (int i = 0; i < num_threads; i++) + { + threads.emplace_back(new std::thread(optimize_pass, + std::ref(datafiles.at(i)), + std::ref(sizes.at(i)), + std::ref(rnds.at(i)), + verbose)); + } + + for (int i = 0; i < num_threads; i++) + { + threads.at(i)->join(); + } + + int best = std::min_element(sizes.begin(), sizes.end()) - sizes.begin(); + size = sizes.at(best); + datafile = datafiles.at(best); +} + +// Go through all the dictionary entries and check what it costs to remove +// them. Removes any entries with negative or zero score. +void update_scores(DataFile &datafile, bool verbose) +{ + size_t oldsize = get_encoded_size(datafile); + + for (size_t i = 0; i < DataFile::dictionarysize; i++) + { + DataFile trial = datafile; + DataFile::dictentry_t dummy = {}; + trial.SetDictionaryEntry(i, dummy); + size_t newsize = get_encoded_size(trial); + + DataFile::dictentry_t d = datafile.GetDictionaryEntry(i); + d.score = newsize - oldsize; + + if (d.score > 0) + { + datafile.SetDictionaryEntry(i, d); + } + else + { + datafile.SetDictionaryEntry(i, dummy); + + if (verbose && d.replacement.size() != 0) + std::cout << "update_scores: dropped " << i + << " score " << -d.score << std::endl; + } + } +} + +void init_dictionary(DataFile &datafile) +{ + rnd_t rnd(datafile.GetSeed()); + + if (datafile.GetGlyphCount() == 0) + return; + + std::set seen_substrings; + std::set added_substrings; + + size_t i = 0; + while (i < DataFile::dictionarysize) + { + DataFile::pixels_t substring = *random_substring(datafile, rnd); + + if (!seen_substrings.count(substring)) + { + seen_substrings.insert(substring); + } + else if (!added_substrings.count(substring)) + { + // When we see a substring second time, add it. + DataFile::dictentry_t d; + d.score = 0; + d.replacement = substring; + datafile.SetDictionaryEntry(i, d); + i++; + added_substrings.insert(substring); + } + } +} + +void optimize(DataFile &datafile, size_t iterations) +{ + bool verbose = false; + rnd_t rnd(datafile.GetSeed()); + + update_scores(datafile, verbose); + + size_t size = get_encoded_size(datafile); + + for (size_t i = 0; i < iterations; i++) + { + optimize_parallel(datafile, size, rnd, verbose); + } + + std::uniform_int_distribution dist(0, std::numeric_limits::max()); + datafile.SetSeed(dist(rnd)); +} + +}} diff --git a/tools/mcufontencoder/src/optimize_rlefont.hh b/tools/mcufontencoder/src/optimize_rlefont.hh new file mode 100644 index 00000000..e4f7c78f --- /dev/null +++ b/tools/mcufontencoder/src/optimize_rlefont.hh @@ -0,0 +1,15 @@ +// This implements the actual optimization passes of the compressor. + +#include "datafile.hh" + +namespace mcufont { +namespace rlefont { + +// Initialize the dictionary table with reasonable guesses. +void init_dictionary(DataFile &datafile); + +// Perform a single optimization step, consisting itself of multiple passes +// of each of the optimization algorithms. +void optimize(DataFile &datafile, size_t iterations = 50); + +}}