Add the mcufont encoder to the tools (including a win32 build) with binaries

ugfx_release_2.6
inmarket 2015-11-22 18:42:11 +10:00
parent 3ae120406e
commit 0ec1a5e4da
29 changed files with 3694 additions and 0 deletions

View File

@ -0,0 +1,2 @@
freetype-2.6.1 source code has not been included in this repository so that you are fully
aware of the license types and restrictions before using it with uGFX.

Binary file not shown.

View File

@ -0,0 +1,43 @@
CXXFLAGS = -O2 -Wall -Werror -Wno-unused-function -Wno-sign-compare -std=c++0x
CXXFLAGS += -ggdb
LDFLAGS += -pthread
# Libfreetype
CXXFLAGS += $(shell freetype-config --cflags)
LDFLAGS += $(shell freetype-config --libs)
# Class to represent font data internally
OBJS = datafile.o
# Utility functions
OBJS += importtools.o exporttools.o
# Import formats
OBJS += bdf_import.o freetype_import.o
# rlefont export format
OBJS += encode_rlefont.o optimize_rlefont.o export_rlefont.o
# bwfont export format
OBJS += export_bwfont.o
all: run_unittests mcufont
clean:
rm -f unittests unittests.cc mcufont $(OBJS)
mcufont: main.o $(OBJS)
g++ $(CXXFLAGS) -o $@ $^ $(LDFLAGS)
unittests.cc: *.hh
cxxtestgen --have-eh --error-printer -o unittests.cc $^
unittests: unittests.cc $(OBJS)
g++ $(CXXFLAGS) -o $@ $^ $(LDFLAGS)
run_unittests: unittests
./unittests
%.o: %.cc *.hh
g++ $(CXXFLAGS) -c $<

View File

@ -0,0 +1,44 @@
ARCH = i686-pc-mingw32-
CXXFLAGS = -O2 -Wall -Werror -Wno-unused-function -Wno-sign-compare -std=c++0x
CXXFLAGS += -ggdb
LDFLAGS += -pthread --static
# Libfreetype
CXXFLAGS += $(shell freetype-config --cflags)
LDFLAGS += $(shell freetype-config --libs)
#FREETYPE2_LIB = ../ugfx/3rdparty/freetype-2.6.1
#CXXFLAGS += -I$(FREETYPE2_LIB)/include
#LDFLAGS += -I$(FREETYPE2_LIB)/lib -lfreetype
# compiler fixes for mingw32
CXXFLAGS += -DNEED_STRING_FIXES -DNEED_THREAD_FIXES
# Class to represent font data internally
OBJS = datafile.o
# Utility functions
OBJS += importtools.o exporttools.o
# Import formats
OBJS += bdf_import.o freetype_import.o
# rlefont export format
OBJS += encode_rlefont.o optimize_rlefont.o export_rlefont.o
# bwfont export format
OBJS += export_bwfont.o
all: mcufont
strip: mcufont.exe
strip mcufont.exe
clean:
rm -f mcufont $(OBJS)
mcufont: main.o $(OBJS)
$(ARCH)g++ $(CXXFLAGS) -o $@ $^ $(LDFLAGS)
%.o: %.cc *.hh
$(ARCH)g++ $(CXXFLAGS) -c $<

View File

@ -0,0 +1,156 @@
#include "bdf_import.hh"
#include "importtools.hh"
#include <sstream>
#include <string>
#include <cctype>
#include <stdexcept>
namespace mcufont {
static std::string toupper(const std::string &input)
{
std::string result;
for (char c: input) result.push_back(::toupper(c));
return result;
}
static int hextoint(char c)
{
if (c >= '0' && c <= '9') return c - '0';
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
throw std::domain_error("Hex digit not in range");
}
static void parse_fontinfo(std::istream &file, DataFile::fontinfo_t &fontinfo)
{
std::string line;
while (std::getline(file, line))
{
std::istringstream s(line);
std::string tag;
s >> tag;
tag = toupper(tag);
if (tag == "FONT")
{
while (isspace(s.peek())) s.get();
std::getline(s, fontinfo.name);
}
else if (tag == "FONTBOUNDINGBOX")
{
int x, y;
s >> fontinfo.max_width >> fontinfo.max_height;
s >> x >> y;
fontinfo.baseline_x = - x;
fontinfo.baseline_y = fontinfo.max_height + y;
}
else if (tag == "STARTCHAR")
{
break;
}
}
}
static bool parse_glyph(std::istream &file, DataFile::glyphentry_t &glyph,
const DataFile::fontinfo_t &fontinfo)
{
glyph.chars.clear();
glyph.width = 0;
// Initialize the character contents to all 0 with proper size.
glyph.data.clear();
glyph.data.resize(fontinfo.max_width * fontinfo.max_height, 0);
int bbx_w = fontinfo.max_width;
int bbx_h = fontinfo.max_height;
int bbx_x = - fontinfo.baseline_x;
int bbx_y = fontinfo.baseline_y - fontinfo.max_height;
// Read glyph metadata
std::string line;
std::string tag;
while (std::getline(file, line))
{
std::istringstream s(line);
s >> tag;
tag = toupper(tag);
if (tag == "ENCODING")
{
int c;
s >> c;
glyph.chars.push_back(c);
}
else if (tag == "DWIDTH")
{
s >> glyph.width;
}
else if (tag == "BBX")
{
s >> bbx_w >> bbx_h >> bbx_x >> bbx_y;
}
else if (tag == "BITMAP")
{
break;
}
}
if (tag != "BITMAP")
return false;
// Read glyph bits
int x0 = fontinfo.baseline_x + bbx_x;
int y = fontinfo.baseline_y - bbx_y - bbx_h;
for (int i = 0; i < bbx_h; i++)
{
std::getline(file, line);
line = toupper(line);
for (int x = 0; x < bbx_w; x++)
{
int nibble = hextoint(line.at(x / 4));
uint8_t pixel = 0;
if (nibble & (8 >> (x % 4)))
pixel = 15;
glyph.data.at(y * fontinfo.max_width + x0 + x) = pixel;
}
y++;
}
std::getline(file, line);
line = toupper(line);
if (line.compare(0, 7, "ENDCHAR") == 0)
return true;
else
return false;
}
std::unique_ptr<DataFile> LoadBDF(std::istream &file)
{
DataFile::fontinfo_t fontinfo = {};
std::vector<DataFile::glyphentry_t> glyphtable;
std::vector<DataFile::dictentry_t> dictionary;
parse_fontinfo(file, fontinfo);
while (file)
{
DataFile::glyphentry_t glyph = {};
if (parse_glyph(file, glyph, fontinfo))
glyphtable.push_back(glyph);
}
eliminate_duplicates(glyphtable);
crop_glyphs(glyphtable, fontinfo);
detect_flags(glyphtable, fontinfo);
fontinfo.line_height = fontinfo.max_height;
std::unique_ptr<DataFile> result(new DataFile(
dictionary, glyphtable, fontinfo));
return result;
}
}

View File

@ -0,0 +1,80 @@
// Function for importing .BDF fonts as data files.
#pragma once
#include "datafile.hh"
namespace mcufont
{
std::unique_ptr<DataFile> LoadBDF(std::istream &file);
}
#ifdef CXXTEST_RUNNING
#include <cxxtest/TestSuite.h>
using namespace mcufont;
class BDFTests: public CxxTest::TestSuite
{
public:
void testLoadBDF()
{
std::istringstream s(testfile);
std::unique_ptr<DataFile> f = LoadBDF(s);
TS_ASSERT_EQUALS(f->GetFontInfo().name, "-Misc-Fixed-Medium-R-Normal--14-130-75-75-C-70-ISO8859-15");
TS_ASSERT_EQUALS(f->GetFontInfo().max_width, 6);
TS_ASSERT_EQUALS(f->GetFontInfo().max_height, 11);
TS_ASSERT_EQUALS(f->GetGlyphCount(), 1);
TS_ASSERT_EQUALS(f->GetGlyphEntry(0).chars.size(), 2);
}
private:
static constexpr const char *testfile =
"STARTFONT 2.1\n"
"FONT -Misc-Fixed-Medium-R-Normal--14-130-75-75-C-70-ISO8859-15\n"
"FONTBOUNDINGBOX 7 14 0 -2\n"
"STARTCHAR defaultchar\n"
"ENCODING 0\n"
"DWIDTH 7 0\n"
"BBX 7 14 0 -2\n"
"BITMAP\n"
"00\n"
"B4\n"
"84\n"
"00\n"
"84\n"
"84\n"
"00\n"
"84\n"
"84\n"
"00\n"
"84\n"
"B4\n"
"00\n"
"00\n"
"ENDCHAR\n"
"STARTCHAR copychar\n"
"ENCODING 2\n"
"DWIDTH 7 0\n"
"BBX 7 14 0 -2\n"
"BITMAP\n"
"00\n"
"B4\n"
"84\n"
"00\n"
"84\n"
"84\n"
"00\n"
"84\n"
"84\n"
"00\n"
"84\n"
"B4\n"
"00\n"
"00\n"
"ENDCHAR\n";
};
#endif

View File

@ -0,0 +1,152 @@
#ifdef NEED_STRING_FIXES
#include <string>
#include <sstream>
#include <stdexcept>
#include <limits>
#include <cstdlib>
namespace std {
template <typename T> inline std::string to_string(T value)
{
std::ostringstream os ;
os << value ;
return os.str() ;
}
inline int stoi( const std::string& str, std::size_t* pos = 0, int base = 10 )
{
const char* begin = str.c_str() ;
char* end = nullptr ;
long value = std::strtol( begin, &end, base ) ;
if( errno == ERANGE || value > std::numeric_limits<int>::max() )
throw std::out_of_range( "stoi: out ofrange" ) ;
if( end == str.c_str() )
throw std::invalid_argument( "stoi: invalid argument" ) ;
if(pos) *pos = end - begin ;
return value ;
}
}
#endif
#ifdef NEED_THREAD_FIXES
#ifndef WIN32STDTHREAD_H
#define WIN32STDTHREAD_H
#include <windows.h>
#include <functional>
#include <memory>
#include <chrono>
#include <system_error>
#include <process.h>
#define _STD_THREAD_INVALID_HANDLE 0
namespace std
{
class thread
{
public:
class id
{
DWORD mId;
void clear() {mId = 0;}
friend class thread;
public:
id(DWORD aId=0):mId(aId){}
bool operator==(const id& other) const {return mId == other.mId;}
};
protected:
HANDLE mHandle;
id mThreadId;
public:
typedef HANDLE native_handle_type;
id get_id() const noexcept {return mThreadId;}
native_handle_type native_handle() const {return mHandle;}
thread(): mHandle(_STD_THREAD_INVALID_HANDLE){}
thread(thread& other)
:mHandle(other.mHandle), mThreadId(other.mThreadId)
{
other.mHandle = _STD_THREAD_INVALID_HANDLE;
other.mThreadId.clear();
}
template<class Function, class... Args>
explicit thread(Function&& f, Args&&... args)
{
typedef decltype(std::bind(f, args...)) Call;
Call* call = new Call(std::bind(f, args...));
mHandle = (HANDLE)_beginthreadex(NULL, 0, threadfunc<Call>,
(LPVOID)call, 0, (unsigned*)&(mThreadId.mId));
}
template <class Call>
static unsigned int __stdcall threadfunc(void* arg)
{
std::unique_ptr<Call> upCall(static_cast<Call*>(arg));
(*upCall)();
return (unsigned long)0;
}
bool joinable() const {return mHandle != _STD_THREAD_INVALID_HANDLE;}
void join()
{
if (get_id() == GetCurrentThreadId())
throw system_error(EDEADLK, generic_category());
if (mHandle == _STD_THREAD_INVALID_HANDLE)
throw system_error(ESRCH, generic_category());
if (!joinable())
throw system_error(EINVAL, generic_category());
WaitForSingleObject(mHandle, INFINITE);
CloseHandle(mHandle);
mHandle = _STD_THREAD_INVALID_HANDLE;
mThreadId.clear();
}
~thread()
{
if (joinable())
std::terminate();
}
thread& operator=(const thread&) = delete;
thread& operator=(thread&& other) noexcept
{
if (joinable())
std::terminate();
swap(std::forward<thread>(other));
return *this;
}
void swap(thread&& other) noexcept
{
std::swap(mHandle, other.mHandle);
std::swap(mThreadId.mId, other.mThreadId.mId);
}
static unsigned int hardware_concurrency() noexcept {return 1;}
void detach()
{
if (!joinable())
throw system_error();
mHandle = _STD_THREAD_INVALID_HANDLE;
mThreadId.clear();
}
};
namespace this_thread
{
inline thread::id get_id() {return thread::id(GetCurrentThreadId());}
inline void yield() {Sleep(0);}
template< class Rep, class Period >
void sleep_for( const std::chrono::duration<Rep,Period>& sleep_duration)
{
Sleep(chrono::duration_cast<chrono::milliseconds>(sleep_duration).count());
}
template <class Clock, class Duration>
void sleep_until(const std::chrono::time_point<Clock,Duration>& sleep_time)
{
sleep_for(sleep_time-Clock::now());
}
}
}
#endif
#endif

View File

@ -0,0 +1,239 @@
#include "datafile.hh"
#include <sstream>
#include <algorithm>
#include <cctype>
#include <stdexcept>
#include "ccfixes.hh"
#define DATAFILE_FORMAT_VERSION 1
namespace mcufont {
DataFile::DataFile(const std::vector<dictentry_t> &dictionary,
const std::vector<glyphentry_t> &glyphs,
const fontinfo_t &fontinfo):
m_dictionary(dictionary), m_glyphtable(glyphs), m_fontinfo(fontinfo)
{
dictentry_t dummy = {};
while (m_dictionary.size() < dictionarysize)
m_dictionary.push_back(dummy);
UpdateLowScoreIndex();
}
void DataFile::Save(std::ostream &file) const
{
file << "Version " << DATAFILE_FORMAT_VERSION << std::endl;
file << "FontName " << m_fontinfo.name << std::endl;
file << "MaxWidth " << m_fontinfo.max_width << std::endl;
file << "MaxHeight " << m_fontinfo.max_height << std::endl;
file << "BaselineX " << m_fontinfo.baseline_x << std::endl;
file << "BaselineY " << m_fontinfo.baseline_y << std::endl;
file << "LineHeight " << m_fontinfo.line_height << std::endl;
file << "Flags " << m_fontinfo.flags << std::endl;
file << "RandomSeed " << m_seed << std::endl;
for (const dictentry_t &d : m_dictionary)
{
if (d.replacement.size() != 0)
{
file << "DictEntry " << d.score << " ";
file << d.ref_encode << " " << d.replacement << std::endl;
}
}
for (const glyphentry_t &g : m_glyphtable)
{
file << "Glyph ";
for (size_t i = 0; i < g.chars.size(); i++)
{
if (i != 0) file << ',';
file << g.chars.at(i);
}
file << " " << g.width << " " << g.data << std::endl;
}
}
std::unique_ptr<DataFile> DataFile::Load(std::istream &file)
{
fontinfo_t fontinfo = {};
std::vector<dictentry_t> dictionary;
std::vector<glyphentry_t> glyphtable;
uint32_t seed = 1234;
int version = -1;
std::string line;
while (std::getline(file, line))
{
std::istringstream input(line);
std::string tag;
input >> tag;
if (tag == "Version")
{
input >> version;
}
else if (tag == "FontName")
{
while (std::isspace(input.peek())) input.get();
std::getline(input, fontinfo.name);
}
else if (tag == "MaxWidth")
{
input >> fontinfo.max_width;
}
else if (tag == "MaxHeight")
{
input >> fontinfo.max_height;
}
else if (tag == "BaselineX")
{
input >> fontinfo.baseline_x;
}
else if (tag == "BaselineY")
{
input >> fontinfo.baseline_y;
}
else if (tag == "LineHeight")
{
input >> fontinfo.line_height;
}
else if (tag == "RandomSeed")
{
input >> seed;
}
else if (tag == "Flags")
{
input >> fontinfo.flags;
}
else if (tag == "DictEntry" && dictionary.size() < dictionarysize)
{
dictentry_t d = {};
input >> d.score >> d.ref_encode >> d.replacement;
dictionary.push_back(d);
}
else if (tag == "Glyph")
{
glyphentry_t g = {};
std::string chars;
input >> chars >> g.width >> g.data;
if ((int)g.data.size() != fontinfo.max_width * fontinfo.max_height)
throw std::runtime_error("wrong glyph data length: " + std::to_string(g.data.size()));
size_t pos = 0;
while (pos < chars.size()) {
size_t p;
g.chars.push_back(std::stoi(chars.substr(pos), &p));
pos += p + 1;
}
glyphtable.push_back(g);
}
}
if (version != DATAFILE_FORMAT_VERSION)
{
return std::unique_ptr<DataFile>(nullptr);
}
std::unique_ptr<DataFile> result(new DataFile(dictionary, glyphtable, fontinfo));
result->SetSeed(seed);
return result;
}
void DataFile::SetDictionaryEntry(size_t index, const dictentry_t &value)
{
m_dictionary.at(index) = value;
if (index == m_lowscoreindex ||
m_dictionary.at(m_lowscoreindex).score > value.score)
{
UpdateLowScoreIndex();
}
}
std::map<size_t, size_t> DataFile::GetCharToGlyphMap() const
{
std::map<size_t, size_t> char_to_glyph;
for (size_t i = 0; i < m_glyphtable.size(); i++)
{
for (size_t c: m_glyphtable[i].chars)
{
char_to_glyph[c] = i;
}
}
return char_to_glyph;
}
std::string DataFile::GlyphToText(size_t index) const
{
std::ostringstream os;
const char glyphchars[] = "....,,,,----XXXX";
for (int y = 0; y < m_fontinfo.max_height; y++)
{
for (int x = 0; x < m_fontinfo.max_width; x++)
{
size_t pos = y * m_fontinfo.max_width + x;
os << glyphchars[m_glyphtable.at(index).data.at(pos)];
}
os << std::endl;
}
return os.str();
}
void DataFile::UpdateLowScoreIndex()
{
auto comparison = [](const dictentry_t &a, const dictentry_t &b)
{
return a.score < b.score;
};
auto iter = std::min_element(m_dictionary.begin(),
m_dictionary.end(),
comparison);
m_lowscoreindex = iter - m_dictionary.begin();
}
std::ostream& operator<<(std::ostream& os, const DataFile::pixels_t& str)
{
for (uint8_t p: str)
{
if (p <= 9)
os << (char)(p + '0');
else if (p <= 15)
os << (char)(p - 10 + 'A');
else
throw std::logic_error("invalid pixel alpha: " + std::to_string(p));
}
return os;
}
std::istream& operator>>(std::istream& is, DataFile::pixels_t& str)
{
char c;
str.clear();
while (isspace(is.peek())) is.get();
while (is.get(c))
{
if (c >= '0' && c <= '9')
str.push_back(c - '0');
else if (c >= 'A' && c <= 'F')
str.push_back(c - 'A' + 10);
else
break;
}
return is;
}
}

View File

@ -0,0 +1,174 @@
// Class to store the data of a font while it is being processed.
// This class can be safely cloned using the default copy constructor.
#pragma once
#include <cstdint>
#include <vector>
#include <string>
#include <fstream>
#include <memory>
#include <map>
namespace mcufont
{
class DataFile
{
public:
typedef std::vector<uint8_t> pixels_t;
struct dictentry_t
{
pixels_t replacement; // The expanded version of this block.
int score; // Number of bytes that having this entry saves.
bool ref_encode; // Encode using references to other dictionary entries.
dictentry_t(): score(0), ref_encode(false) {}
};
struct glyphentry_t
{
pixels_t data; // The full data of the glyph.
std::vector<int> chars; // Characters that this glyph represents.
int width; // Tracking width of the character.
};
struct fontinfo_t
{
std::string name; // Name of the typeface
int max_width; // Width of the character bounding box.
int max_height; // Height of the character bounding box.
int baseline_x; // X coordinate (from left) of the baseline.
int baseline_y; // Y coordinate (from top) of the baseline.
int line_height; // Line height (vertical advance).
int flags;
};
static const int FLAG_MONOSPACE = 0x01;
static const int FLAG_BW = 0x02;
// Construct from data in memory.
DataFile(const std::vector<dictentry_t> &dictionary,
const std::vector<glyphentry_t> &glyphs,
const fontinfo_t &fontinfo);
// Save to a file (custom format)
void Save(std::ostream &file) const;
// Load from a file (custom format)
// Returns nullptr if load fails.
static std::unique_ptr<DataFile> Load(std::istream &file);
// Get or set an entry in the dictionary. The size of the dictionary
// is constant. Entries 0 to 23 are reserved for special purposes.
static const size_t dictionarysize = 256 - 24;
const dictentry_t &GetDictionaryEntry(size_t index) const
{ return m_dictionary.at(index); }
void SetDictionaryEntry(size_t index, const dictentry_t &value);
const std::vector<dictentry_t> &GetDictionary() const
{ return m_dictionary; }
// Get the index of the dictionary entry with the lowest score.
size_t GetLowScoreIndex() const
{ return m_lowscoreindex; }
// Get an entry in the glyph table.
size_t GetGlyphCount() const
{ return m_glyphtable.size(); }
const glyphentry_t &GetGlyphEntry(size_t index) const
{ return m_glyphtable.at(index); }
const std::vector<glyphentry_t> &GetGlyphTable() const
{ return m_glyphtable; }
// Create a map of char indices to glyph indices
std::map<size_t, size_t> GetCharToGlyphMap() const;
// Get the information that applies to all glyphs.
const fontinfo_t &GetFontInfo() const
{ return m_fontinfo; }
// Show a glyph as text.
std::string GlyphToText(size_t index) const;
// Get the random generator seed
// The seed is stored in the datafile to get deterministic behaviour
// for debugging.
uint32_t GetSeed() const { return m_seed; }
void SetSeed(uint32_t seed) { m_seed = seed; }
private:
std::vector<dictentry_t> m_dictionary;
std::vector<glyphentry_t> m_glyphtable;
fontinfo_t m_fontinfo;
uint32_t m_seed;
size_t m_lowscoreindex;
void UpdateLowScoreIndex();
};
std::ostream& operator<<(std::ostream& os, const DataFile::pixels_t& str);
std::istream& operator>>(std::istream& is, DataFile::pixels_t& str);
}
#ifdef CXXTEST_RUNNING
#include <cxxtest/TestSuite.h>
using namespace mcufont;
class DataFileTests: public CxxTest::TestSuite
{
public:
void testFileLoad()
{
std::istringstream s(testfile);
std::unique_ptr<DataFile> f = DataFile::Load(s);
TS_ASSERT_EQUALS(f->GetFontInfo().name, "Sans Serif");
TS_ASSERT_EQUALS(f->GetFontInfo().max_width, 4);
TS_ASSERT_EQUALS(f->GetFontInfo().max_height, 6);
TS_ASSERT_EQUALS(f->GetDictionaryEntry(0).score, 5);
TS_ASSERT_EQUALS(f->GetDictionaryEntry(1).score, 13);
TS_ASSERT_EQUALS(f->GetGlyphCount(), 3);
DataFile::pixels_t expected = {
0,15,0,15, 0,15,0,15, 0,15,0,15, 0,15,0,15, 0,15,0,15, 0,15,0,15
};
TS_ASSERT_EQUALS(f->GetGlyphEntry(0).data.size(), 24);
TS_ASSERT(f->GetGlyphEntry(0).data == expected);
}
void testFileSave()
{
std::istringstream is1(testfile);
std::unique_ptr<DataFile> f1 = DataFile::Load(is1);
std::ostringstream os;
f1->Save(os);
std::string text = os.str();
std::istringstream is2(text);
std::unique_ptr<DataFile> f2 = DataFile::Load(is2);
TS_ASSERT_EQUALS(f1->GetFontInfo().name, f2->GetFontInfo().name);
TS_ASSERT(f1->GetGlyphEntry(0).data == f2->GetGlyphEntry(0).data);
}
private:
static constexpr const char *testfile =
"Version 1\n"
"FontName Sans Serif\n"
"MaxWidth 4\n"
"MaxHeight 6\n"
"BaselineX 1\n"
"BaselineY 1\n"
"DictEntry 5 0 0F0F0\n"
"DictEntry 13 0 F0F0F0\n"
"DictEntry 1 0 F0F0F0\n"
"Glyph 1,2,3 4 0F0F0F0F0F0F0F0F0F0F0F0F\n"
"Glyph 4 4 0F0F0F0F0F0F0F0F0F0F0F0F\n"
"Glyph 5 4 0F0F0F0F0F0F0F0F0F0F0F0F\n";
};
#endif

View File

@ -0,0 +1,735 @@
#include "encode_rlefont.hh"
#include <algorithm>
#include <stdexcept>
#include "ccfixes.hh"
// Number of reserved codes before the dictionary entries.
#define DICT_START 24
// Special reference to mean "fill with zeros to the end of the glyph"
#define REF_FILLZEROS 16
// RLE codes
#define RLE_CODEMASK 0xC0
#define RLE_VALMASK 0x3F
#define RLE_ZEROS 0x00 // 0 to 63 zeros
#define RLE_64ZEROS 0x40 // (1 to 64) * 64 zeros
#define RLE_ONES 0x80 // 1 to 64 full alphas
#define RLE_SHADE 0xC0 // 1 to 4 partial alphas
// Dictionary "fill entries" for encoding bits directly.
#define DICT_START7BIT 4
#define DICT_START6BIT 132
#define DICT_START5BIT 196
#define DICT_START4BIT 228
#define DICT_START3BIT 244
#define DICT_START2BIT 252
namespace mcufont {
namespace rlefont {
// Get bit count for the "fill entries"
static size_t fillentry_bitcount(size_t index)
{
if (index >= DICT_START2BIT)
return 2;
else if (index >= DICT_START3BIT)
return 3;
else if (index >= DICT_START4BIT)
return 4;
else if (index >= DICT_START5BIT)
return 5;
else if (index >= DICT_START6BIT)
return 6;
else
return 7;
}
// Count the number of equal pixels at the beginning of the pixelstring.
static size_t prefix_length(const DataFile::pixels_t &pixels, size_t pos)
{
uint8_t pixel = pixels.at(pos);
size_t count = 1;
while (pos + count < pixels.size() &&
pixels.at(pos + count) == pixel)
{
count++;
}
return count;
}
// Perform the RLE encoding for a dictionary entry.
static encoded_font_t::rlestring_t encode_rle(const DataFile::pixels_t &pixels)
{
encoded_font_t::rlestring_t result;
size_t pos = 0;
while (pos < pixels.size())
{
uint8_t pixel = pixels.at(pos);
size_t count = prefix_length(pixels, pos);
pos += count;
if (pixel == 0)
{
// Up to 63 zeros can be encoded with RLE_ZEROS. If there are more,
// encode using RLE_64ZEROS, and then whatever remains with RLE_ZEROS.
while (count >= 64)
{
size_t c = (count > 4096) ? 64 : (count / 64);
result.push_back(RLE_64ZEROS | (c - 1));
count -= c * 64;
}
if (count)
{
result.push_back(RLE_ZEROS | count);
}
}
else if (pixel == 15)
{
// Encode ones.
while (count)
{
size_t c = (count > 64) ? 64 : count;
result.push_back(RLE_ONES | (c - 1));
count -= c;
}
}
else
{
// Encode shades.
while (count)
{
size_t c = (count > 4) ? 4 : count;
result.push_back(RLE_SHADE | ((c - 1) << 4) | pixel);
count -= c;
}
}
}
return result;
}
// We use a tree structure to represent the dictionary entries.
// Using this tree, we can perform a combined Aho-Corasick string matching
// and breadth-first search to find the optimal encoding of glyph data.
class DictTreeNode
{
public:
constexpr DictTreeNode():
m_index(-1),
m_ref(false),
m_length(0),
m_child0(nullptr),
m_child15(nullptr),
m_suffix(nullptr)
{}
void SetChild(uint8_t p, DictTreeNode *child)
{
if (p == 0)
m_child0 = child;
else if (p == 15)
m_child15 = child;
else if (p > 15)
throw std::logic_error("invalid pixel alpha: " + std::to_string(p));
else
{
if (!m_children)
{
m_children.reset(new DictTreeNode*[14]());
}
m_children[p - 1] = child;
}
}
DictTreeNode* GetChild(uint8_t p) const
{
if (p == 0)
return m_child0;
else if (p == 15)
return m_child15;
else if (p > 15)
throw std::logic_error("invalid pixel alpha: " + std::to_string(p));
else if (!m_children)
return nullptr;
else
return m_children[p - 1];
}
bool HasIntermediateChildren() const { return m_children != nullptr; }
int GetIndex() const { return m_index; }
void SetIndex(int index) { m_index = index; }
bool GetRef() const { return m_ref; }
void SetRef(bool ref) { m_ref = ref; }
size_t GetLength() const { return m_length; }
void SetLength(size_t length) { m_length = length; }
DictTreeNode *GetSuffix() const { return m_suffix; }
void SetSuffix(DictTreeNode *suffix) { m_suffix = suffix; }
private:
// Index of dictionary entry or -1 if just a intermediate node.
int m_index;
// True for ref-encoded dictionary entries. Used to avoid recursion when
// encoding them.
bool m_ref;
// Length of the corresponding dictionary entry replacement.
// Equals the distance from the tree root.
size_t m_length;
// Most tree nodes will only ever contains children for 0 or 15.
// Therefore the array for other nodes is allocated only on demand.
DictTreeNode *m_child0;
DictTreeNode *m_child15;
std::unique_ptr<DictTreeNode*[]> m_children;
// Pointer to the longest suffix of this entry that exists in the
// dictionary.
DictTreeNode *m_suffix;
};
// Preallocated array for tree nodes
class TreeAllocator
{
public:
TreeAllocator(size_t count)
{
m_storage.reset(new DictTreeNode[count]);
m_next = m_storage.get();
m_left = count;
}
DictTreeNode *allocate()
{
if (m_left == 0)
throw std::logic_error("Ran out of preallocated entries");
m_left--;
return m_next++;
}
private:
std::unique_ptr<DictTreeNode[]> m_storage;
DictTreeNode *m_next;
size_t m_left;
};
// Add a new dictionary entry to the tree. Adds the intermediate nodes, but
// does not yet fill the suffix pointers.
static DictTreeNode* add_tree_entry(const DataFile::pixels_t &entry, int index,
bool ref_encoded, DictTreeNode *root,
TreeAllocator &storage)
{
DictTreeNode* node = root;
for (uint8_t p : entry)
{
DictTreeNode* branch = node->GetChild(p);
if (!branch)
{
branch = storage.allocate();
node->SetChild(p, branch);
}
node = branch;
}
// Replace the entry if it either does not yet have an encoding, or if
// the new entry is non-ref (i.e. can be used in more situations).
if (node->GetIndex() < 0 || (node->GetRef() && !ref_encoded))
{
node->SetIndex(index);
node->SetRef(ref_encoded);
node->SetLength(entry.size());
}
return node;
}
// Walk the tree and find if the entry exists in the tree. If it does,
// returns a pointer to it, otherwise nullptr.
static DictTreeNode *find_tree_node(DataFile::pixels_t::const_iterator begin,
DataFile::pixels_t::const_iterator end,
DictTreeNode *root)
{
DictTreeNode* node = root;
while (begin != end)
{
uint8_t pixel = *begin++;
node = node->GetChild(pixel);
if (!node)
return nullptr;
}
return node;
}
// Fill in the suffix pointers recursively for the given subtree.
static void fill_tree_suffixes(DictTreeNode *root, DictTreeNode *subtree,
const DataFile::pixels_t &entry)
{
for (size_t i = 1; i < entry.size(); i++)
{
DictTreeNode *node = find_tree_node(entry.begin() + i, entry.end(), root);
if (node)
{
subtree->SetSuffix(node);
break;
}
}
if (!subtree->GetSuffix())
subtree->SetSuffix(root);
DataFile::pixels_t newentry(entry);
newentry.resize(entry.size() + 1);
for (uint8_t i = 0; i < 16; i++)
{
// Speed-up for the common case of 0 and 15 alphas.
if (i == 1 && !subtree->HasIntermediateChildren())
i += 14;
DictTreeNode *child = subtree->GetChild(i);
if (child)
{
newentry.at(entry.size()) = i;
fill_tree_suffixes(root, child, newentry);
}
}
}
// Construct a lookup tree from the dictionary entries.
static DictTreeNode* construct_tree(const std::vector<DataFile::dictentry_t> &dictionary,
TreeAllocator &storage, bool fast)
{
DictTreeNode* root = storage.allocate();
// Populate the hardcoded entries for 0 to 15 alpha.
for (int j = 0; j < 16; j++)
{
DictTreeNode *node = storage.allocate();
node->SetIndex(j);
node->SetRef(false);
node->SetLength(1);
root->SetChild(j, node);
}
// Populate the actual dictionary entries
size_t i = DICT_START;
for (DataFile::dictentry_t d : dictionary)
{
if (!d.replacement.size())
break;
add_tree_entry(d.replacement, i, d.ref_encode, root, storage);
i++;
}
if (!fast)
{
// Populate the fill entries for rest of dictionary
for (; i < 256; i++)
{
DataFile::pixels_t pixels;
size_t bitcount = fillentry_bitcount(i);
uint8_t byte = i - DICT_START7BIT;
for (size_t j = 0; j < bitcount; j++)
{
uint8_t p = (byte & (1 << j)) ? 15 : 0;
pixels.push_back(p);
}
add_tree_entry(pixels, i, false, root, storage);
}
// Fill in the suffix pointers for optimal encoding
DataFile::pixels_t nullentry;
fill_tree_suffixes(root, root, nullentry);
}
return root;
}
// Structure for keeping track of the shortest encoding to reach particular
// point of the pixel string.
struct encoding_link_t
{
// Index of the position prior to the last dictionary entry.
size_t previous;
// Index of the dictionary entry that brings us to this point.
int index;
// Number of links to get here from the start of the string.
size_t length;
constexpr encoding_link_t(): previous(0), index(-1), length(9999999) {}
};
// Perform the reference encoding for a glyph entry (optimal version).
// Uses a modified Aho-Corasick algorithm combined with breadth first search
// to find the shortest representation.
static encoded_font_t::refstring_t encode_ref_slow(const DataFile::pixels_t &pixels,
const DictTreeNode *root,
bool is_glyph)
{
// Chain of encodings. Each entry in this array corresponds to a position
// in the pixel string.
std::unique_ptr<encoding_link_t[]> chain(new encoding_link_t[pixels.size() + 1]);
chain[0].previous = 0;
chain[0].index = 0;
chain[0].length = 0;
// Read the pixels one-by-one and update the encoding links accordingly.
const DictTreeNode *node = root;
for (size_t pos = 0; pos < pixels.size(); pos++)
{
uint8_t pixel = pixels.at(pos);
const DictTreeNode *branch = node->GetChild(pixel);
while (!branch)
{
// Cannot expand this sequence, defer to suffix.
node = node->GetSuffix();
branch = node->GetChild(pixel);
}
node = branch;
// We have arrived at a new node, add it and any proper suffixes to
// the link chain.
const DictTreeNode *suffix = node;
while (suffix != root)
{
if (suffix->GetIndex() >= 0 && (is_glyph || !suffix->GetRef()))
{
encoding_link_t link;
link.previous = pos + 1 - suffix->GetLength();
link.index = suffix->GetIndex();
link.length = chain[link.previous].length + 1;
if (link.length < chain[pos + 1].length)
chain[pos + 1] = link;
}
suffix = suffix->GetSuffix();
}
}
// Check if we can shorten the final encoding using REF_FILLZEROS.
if (is_glyph)
{
for (size_t pos = pixels.size() - 1; pos > 0; pos--)
{
if (pixels.at(pos) != 0)
break;
encoding_link_t link;
link.previous = pos;
link.index = REF_FILLZEROS;
link.length = chain[pos].length + 1;
if (link.length <= chain[pixels.size()].length)
chain[pixels.size()] = link;
}
}
// Backtrack from the final link back to the start and construct the
// encoded string.
encoded_font_t::refstring_t result;
size_t len = chain[pixels.size()].length;
result.resize(len);
size_t pos = pixels.size();
for (size_t i = len; i > 0; i--)
{
result.at(i - 1) = chain[pos].index;
pos = chain[pos].previous;
}
return result;
}
// Walk the tree as far as possible following the given pixel string iterator.
// Returns number of pixels encoded, and index is set to the dictionary reference.
static size_t walk_tree(const DictTreeNode *tree,
DataFile::pixels_t::const_iterator pixels,
DataFile::pixels_t::const_iterator pixelsend,
int &index, bool is_glyph)
{
size_t best_length = 0;
size_t length = 0;
index = -1;
const DictTreeNode* node = tree;
while (pixels != pixelsend)
{
uint8_t pixel = *pixels++;
node = node->GetChild(pixel);
if (!node)
break;
length++;
if (is_glyph || !node->GetRef())
{
if (node->GetIndex() >= 0)
{
index = node->GetIndex();
best_length = length;
}
}
}
if (index < 0)
throw std::logic_error("walk_tree failed to find a valid encoding");
return best_length;
}
// Perform the reference encoding for a glyph entry (fast version).
// Uses a simple greedy search to find select the encodings.
static encoded_font_t::refstring_t encode_ref_fast(const DataFile::pixels_t &pixels,
const DictTreeNode *tree,
bool is_glyph)
{
encoded_font_t::refstring_t result;
// Strip any zeroes from end
size_t end = pixels.size();
if (is_glyph)
{
while (end > 0 && pixels.at(end - 1) == 0) end--;
}
size_t i = 0;
while (i < end)
{
int index;
i += walk_tree(tree, pixels.begin() + i, pixels.end(), index, is_glyph);
result.push_back(index);
}
if (i < pixels.size())
result.push_back(REF_FILLZEROS);
return result;
}
static encoded_font_t::refstring_t encode_ref(const DataFile::pixels_t &pixels,
const DictTreeNode *tree,
bool is_glyph, bool fast)
{
if (fast)
return encode_ref_fast(pixels, tree, is_glyph);
else
return encode_ref_slow(pixels, tree, is_glyph);
}
// Compare dictionary entries by their coding type.
// Sorts RLE-encoded entries first and any empty entries last.
static bool cmp_dict_coding(const DataFile::dictentry_t &a,
const DataFile::dictentry_t &b)
{
if (a.replacement.size() == 0 && b.replacement.size() != 0)
return false;
else if (a.replacement.size() != 0 && b.replacement.size() == 0)
return true;
else if (a.ref_encode == false && b.ref_encode == true)
return true;
else
return false;
}
size_t estimate_tree_node_count(const std::vector<DataFile::dictentry_t> &dict)
{
size_t count = DICT_START; // Preallocated entries
for (const DataFile::dictentry_t &d: dict)
{
count += d.replacement.size();
}
count += 128 * 7; // Fill entries
return count;
}
std::unique_ptr<encoded_font_t> encode_font(const DataFile &datafile,
bool fast)
{
std::unique_ptr<encoded_font_t> result(new encoded_font_t);
// Sort the dictionary so that RLE-coded entries come first.
// This way the two are easy to distinguish based on index.
std::vector<DataFile::dictentry_t> sorted_dict = datafile.GetDictionary();
std::stable_sort(sorted_dict.begin(), sorted_dict.end(), cmp_dict_coding);
// Build the binary tree for looking up references.
size_t count = estimate_tree_node_count(sorted_dict);
TreeAllocator allocator(count);
DictTreeNode* tree = construct_tree(sorted_dict, allocator, fast);
// Encode the dictionary entries, using either RLE or reference method.
for (const DataFile::dictentry_t &d : sorted_dict)
{
if (d.replacement.size() == 0)
{
continue;
}
else if (d.ref_encode)
{
result->ref_dictionary.push_back(encode_ref(d.replacement, tree, false, fast));
}
else
{
result->rle_dictionary.push_back(encode_rle(d.replacement));
}
}
// Then reference-encode the glyphs
for (const DataFile::glyphentry_t &g : datafile.GetGlyphTable())
{
result->glyphs.push_back(encode_ref(g.data, tree, true, fast));
}
// Optionally verify that the encoding was correct.
if (!fast)
{
for (size_t i = 0; i < datafile.GetGlyphCount(); i++)
{
std::unique_ptr<DataFile::pixels_t> decoded =
decode_glyph(*result, i, datafile.GetFontInfo());
if (*decoded != datafile.GetGlyphEntry