Browse Source

Add the mcufont encoder to the tools (including a win32 build) with binaries

remotes/origin_old/ugfx_release_2.6
inmarket 6 years ago
parent
commit
0ec1a5e4da
  1. 2
      3rdparty/freetype-2.6.1/readme.txt
  2. BIN
      tools/mcufontencoder/binaries/linux/mcufont
  3. BIN
      tools/mcufontencoder/binaries/windows/freetype6.dll
  4. BIN
      tools/mcufontencoder/binaries/windows/libgcc_s_dw2-1.dll
  5. BIN
      tools/mcufontencoder/binaries/windows/mcufont.exe
  6. BIN
      tools/mcufontencoder/binaries/windows/zlib1.dll
  7. 43
      tools/mcufontencoder/src/Makefile
  8. 44
      tools/mcufontencoder/src/Makefile.mingw32
  9. 156
      tools/mcufontencoder/src/bdf_import.cc
  10. 80
      tools/mcufontencoder/src/bdf_import.hh
  11. 152
      tools/mcufontencoder/src/ccfixes.hh
  12. 239
      tools/mcufontencoder/src/datafile.cc
  13. 174
      tools/mcufontencoder/src/datafile.hh
  14. 735
      tools/mcufontencoder/src/encode_rlefont.cc
  15. 126
      tools/mcufontencoder/src/encode_rlefont.hh
  16. 247
      tools/mcufontencoder/src/export_bwfont.cc
  17. 16
      tools/mcufontencoder/src/export_bwfont.hh
  18. 181
      tools/mcufontencoder/src/export_rlefont.cc
  19. 15
      tools/mcufontencoder/src/export_rlefont.hh
  20. 179
      tools/mcufontencoder/src/exporttools.cc
  21. 52
      tools/mcufontencoder/src/exporttools.hh
  22. 177
      tools/mcufontencoder/src/freetype_import.cc
  23. 10
      tools/mcufontencoder/src/freetype_import.hh
  24. 134
      tools/mcufontencoder/src/importtools.cc
  25. 20
      tools/mcufontencoder/src/importtools.hh
  26. BIN
      tools/mcufontencoder/src/libfreetype.dll.a
  27. 480
      tools/mcufontencoder/src/main.cc
  28. 417
      tools/mcufontencoder/src/optimize_rlefont.cc
  29. 15
      tools/mcufontencoder/src/optimize_rlefont.hh

2
3rdparty/freetype-2.6.1/readme.txt

@ -0,0 +1,2 @@
freetype-2.6.1 source code has not been included in this repository so that you are fully
aware of the license types and restrictions before using it with uGFX.

BIN
tools/mcufontencoder/binaries/linux/mcufont

Binary file not shown.

BIN
tools/mcufontencoder/binaries/windows/freetype6.dll

Binary file not shown.

BIN
tools/mcufontencoder/binaries/windows/libgcc_s_dw2-1.dll

Binary file not shown.

BIN
tools/mcufontencoder/binaries/windows/mcufont.exe

Binary file not shown.

BIN
tools/mcufontencoder/binaries/windows/zlib1.dll

Binary file not shown.

43
tools/mcufontencoder/src/Makefile

@ -0,0 +1,43 @@
CXXFLAGS = -O2 -Wall -Werror -Wno-unused-function -Wno-sign-compare -std=c++0x
CXXFLAGS += -ggdb
LDFLAGS += -pthread
# Libfreetype
CXXFLAGS += $(shell freetype-config --cflags)
LDFLAGS += $(shell freetype-config --libs)
# Class to represent font data internally
OBJS = datafile.o
# Utility functions
OBJS += importtools.o exporttools.o
# Import formats
OBJS += bdf_import.o freetype_import.o
# rlefont export format
OBJS += encode_rlefont.o optimize_rlefont.o export_rlefont.o
# bwfont export format
OBJS += export_bwfont.o
all: run_unittests mcufont
clean:
rm -f unittests unittests.cc mcufont $(OBJS)
mcufont: main.o $(OBJS)
g++ $(CXXFLAGS) -o $@ $^ $(LDFLAGS)
unittests.cc: *.hh
cxxtestgen --have-eh --error-printer -o unittests.cc $^
unittests: unittests.cc $(OBJS)
g++ $(CXXFLAGS) -o $@ $^ $(LDFLAGS)
run_unittests: unittests
./unittests
%.o: %.cc *.hh
g++ $(CXXFLAGS) -c $<

44
tools/mcufontencoder/src/Makefile.mingw32

@ -0,0 +1,44 @@
ARCH = i686-pc-mingw32-
CXXFLAGS = -O2 -Wall -Werror -Wno-unused-function -Wno-sign-compare -std=c++0x
CXXFLAGS += -ggdb
LDFLAGS += -pthread --static
# Libfreetype
CXXFLAGS += $(shell freetype-config --cflags)
LDFLAGS += $(shell freetype-config --libs)
#FREETYPE2_LIB = ../ugfx/3rdparty/freetype-2.6.1
#CXXFLAGS += -I$(FREETYPE2_LIB)/include
#LDFLAGS += -I$(FREETYPE2_LIB)/lib -lfreetype
# compiler fixes for mingw32
CXXFLAGS += -DNEED_STRING_FIXES -DNEED_THREAD_FIXES
# Class to represent font data internally
OBJS = datafile.o
# Utility functions
OBJS += importtools.o exporttools.o
# Import formats
OBJS += bdf_import.o freetype_import.o
# rlefont export format
OBJS += encode_rlefont.o optimize_rlefont.o export_rlefont.o
# bwfont export format
OBJS += export_bwfont.o
all: mcufont
strip: mcufont.exe
strip mcufont.exe
clean:
rm -f mcufont $(OBJS)
mcufont: main.o $(OBJS)
$(ARCH)g++ $(CXXFLAGS) -o $@ $^ $(LDFLAGS)
%.o: %.cc *.hh
$(ARCH)g++ $(CXXFLAGS) -c $<

156
tools/mcufontencoder/src/bdf_import.cc

@ -0,0 +1,156 @@
#include "bdf_import.hh"
#include "importtools.hh"
#include <sstream>
#include <string>
#include <cctype>
#include <stdexcept>
namespace mcufont {
static std::string toupper(const std::string &input)
{
std::string result;
for (char c: input) result.push_back(::toupper(c));
return result;
}
static int hextoint(char c)
{
if (c >= '0' && c <= '9') return c - '0';
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
throw std::domain_error("Hex digit not in range");
}
static void parse_fontinfo(std::istream &file, DataFile::fontinfo_t &fontinfo)
{
std::string line;
while (std::getline(file, line))
{
std::istringstream s(line);
std::string tag;
s >> tag;
tag = toupper(tag);
if (tag == "FONT")
{
while (isspace(s.peek())) s.get();
std::getline(s, fontinfo.name);
}
else if (tag == "FONTBOUNDINGBOX")
{
int x, y;
s >> fontinfo.max_width >> fontinfo.max_height;
s >> x >> y;
fontinfo.baseline_x = - x;
fontinfo.baseline_y = fontinfo.max_height + y;
}
else if (tag == "STARTCHAR")
{
break;
}
}
}
static bool parse_glyph(std::istream &file, DataFile::glyphentry_t &glyph,
const DataFile::fontinfo_t &fontinfo)
{
glyph.chars.clear();
glyph.width = 0;
// Initialize the character contents to all 0 with proper size.
glyph.data.clear();
glyph.data.resize(fontinfo.max_width * fontinfo.max_height, 0);
int bbx_w = fontinfo.max_width;
int bbx_h = fontinfo.max_height;
int bbx_x = - fontinfo.baseline_x;
int bbx_y = fontinfo.baseline_y - fontinfo.max_height;
// Read glyph metadata
std::string line;
std::string tag;
while (std::getline(file, line))
{
std::istringstream s(line);
s >> tag;
tag = toupper(tag);
if (tag == "ENCODING")
{
int c;
s >> c;
glyph.chars.push_back(c);
}
else if (tag == "DWIDTH")
{
s >> glyph.width;
}
else if (tag == "BBX")
{
s >> bbx_w >> bbx_h >> bbx_x >> bbx_y;
}
else if (tag == "BITMAP")
{
break;
}
}
if (tag != "BITMAP")
return false;
// Read glyph bits
int x0 = fontinfo.baseline_x + bbx_x;
int y = fontinfo.baseline_y - bbx_y - bbx_h;
for (int i = 0; i < bbx_h; i++)
{
std::getline(file, line);
line = toupper(line);
for (int x = 0; x < bbx_w; x++)
{
int nibble = hextoint(line.at(x / 4));
uint8_t pixel = 0;
if (nibble & (8 >> (x % 4)))
pixel = 15;
glyph.data.at(y * fontinfo.max_width + x0 + x) = pixel;
}
y++;
}
std::getline(file, line);
line = toupper(line);
if (line.compare(0, 7, "ENDCHAR") == 0)
return true;
else
return false;
}
std::unique_ptr<DataFile> LoadBDF(std::istream &file)
{
DataFile::fontinfo_t fontinfo = {};
std::vector<DataFile::glyphentry_t> glyphtable;
std::vector<DataFile::dictentry_t> dictionary;
parse_fontinfo(file, fontinfo);
while (file)
{
DataFile::glyphentry_t glyph = {};
if (parse_glyph(file, glyph, fontinfo))
glyphtable.push_back(glyph);
}
eliminate_duplicates(glyphtable);
crop_glyphs(glyphtable, fontinfo);
detect_flags(glyphtable, fontinfo);
fontinfo.line_height = fontinfo.max_height;
std::unique_ptr<DataFile> result(new DataFile(
dictionary, glyphtable, fontinfo));
return result;
}
}

80
tools/mcufontencoder/src/bdf_import.hh

@ -0,0 +1,80 @@
// Function for importing .BDF fonts as data files.
#pragma once
#include "datafile.hh"
namespace mcufont
{
std::unique_ptr<DataFile> LoadBDF(std::istream &file);
}
#ifdef CXXTEST_RUNNING
#include <cxxtest/TestSuite.h>
using namespace mcufont;
class BDFTests: public CxxTest::TestSuite
{
public:
void testLoadBDF()
{
std::istringstream s(testfile);
std::unique_ptr<DataFile> f = LoadBDF(s);
TS_ASSERT_EQUALS(f->GetFontInfo().name, "-Misc-Fixed-Medium-R-Normal--14-130-75-75-C-70-ISO8859-15");
TS_ASSERT_EQUALS(f->GetFontInfo().max_width, 6);
TS_ASSERT_EQUALS(f->GetFontInfo().max_height, 11);
TS_ASSERT_EQUALS(f->GetGlyphCount(), 1);
TS_ASSERT_EQUALS(f->GetGlyphEntry(0).chars.size(), 2);
}
private:
static constexpr const char *testfile =
"STARTFONT 2.1\n"
"FONT -Misc-Fixed-Medium-R-Normal--14-130-75-75-C-70-ISO8859-15\n"
"FONTBOUNDINGBOX 7 14 0 -2\n"
"STARTCHAR defaultchar\n"
"ENCODING 0\n"
"DWIDTH 7 0\n"
"BBX 7 14 0 -2\n"
"BITMAP\n"
"00\n"
"B4\n"
"84\n"
"00\n"
"84\n"
"84\n"
"00\n"
"84\n"
"84\n"
"00\n"
"84\n"
"B4\n"
"00\n"
"00\n"
"ENDCHAR\n"
"STARTCHAR copychar\n"
"ENCODING 2\n"
"DWIDTH 7 0\n"
"BBX 7 14 0 -2\n"
"BITMAP\n"
"00\n"
"B4\n"
"84\n"
"00\n"
"84\n"
"84\n"
"00\n"
"84\n"
"84\n"
"00\n"
"84\n"
"B4\n"
"00\n"
"00\n"
"ENDCHAR\n";
};
#endif

152
tools/mcufontencoder/src/ccfixes.hh

@ -0,0 +1,152 @@
#ifdef NEED_STRING_FIXES
#include <string>
#include <sstream>
#include <stdexcept>
#include <limits>
#include <cstdlib>
namespace std {
template <typename T> inline std::string to_string(T value)
{
std::ostringstream os ;
os << value ;
return os.str() ;
}
inline int stoi( const std::string& str, std::size_t* pos = 0, int base = 10 )
{
const char* begin = str.c_str() ;
char* end = nullptr ;
long value = std::strtol( begin, &end, base ) ;
if( errno == ERANGE || value > std::numeric_limits<int>::max() )
throw std::out_of_range( "stoi: out ofrange" ) ;
if( end == str.c_str() )
throw std::invalid_argument( "stoi: invalid argument" ) ;
if(pos) *pos = end - begin ;
return value ;
}
}
#endif
#ifdef NEED_THREAD_FIXES
#ifndef WIN32STDTHREAD_H
#define WIN32STDTHREAD_H
#include <windows.h>
#include <functional>
#include <memory>
#include <chrono>
#include <system_error>
#include <process.h>
#define _STD_THREAD_INVALID_HANDLE 0
namespace std
{
class thread
{
public:
class id
{
DWORD mId;
void clear() {mId = 0;}
friend class thread;
public:
id(DWORD aId=0):mId(aId){}
bool operator==(const id& other) const {return mId == other.mId;}
};
protected:
HANDLE mHandle;
id mThreadId;
public:
typedef HANDLE native_handle_type;
id get_id() const noexcept {return mThreadId;}
native_handle_type native_handle() const {return mHandle;}
thread(): mHandle(_STD_THREAD_INVALID_HANDLE){}
thread(thread& other)
:mHandle(other.mHandle), mThreadId(other.mThreadId)
{
other.mHandle = _STD_THREAD_INVALID_HANDLE;
other.mThreadId.clear();
}
template<class Function, class... Args>
explicit thread(Function&& f, Args&&... args)
{
typedef decltype(std::bind(f, args...)) Call;
Call* call = new Call(std::bind(f, args...));
mHandle = (HANDLE)_beginthreadex(NULL, 0, threadfunc<Call>,
(LPVOID)call, 0, (unsigned*)&(mThreadId.mId));
}
template <class Call>
static unsigned int __stdcall threadfunc(void* arg)
{
std::unique_ptr<Call> upCall(static_cast<Call*>(arg));
(*upCall)();
return (unsigned long)0;
}
bool joinable() const {return mHandle != _STD_THREAD_INVALID_HANDLE;}
void join()
{
if (get_id() == GetCurrentThreadId())
throw system_error(EDEADLK, generic_category());
if (mHandle == _STD_THREAD_INVALID_HANDLE)
throw system_error(ESRCH, generic_category());
if (!joinable())
throw system_error(EINVAL, generic_category());
WaitForSingleObject(mHandle, INFINITE);
CloseHandle(mHandle);
mHandle = _STD_THREAD_INVALID_HANDLE;
mThreadId.clear();
}
~thread()
{
if (joinable())
std::terminate();
}
thread& operator=(const thread&) = delete;
thread& operator=(thread&& other) noexcept
{
if (joinable())
std::terminate();
swap(std::forward<thread>(other));
return *this;
}
void swap(thread&& other) noexcept
{
std::swap(mHandle, other.mHandle);
std::swap(mThreadId.mId, other.mThreadId.mId);
}
static unsigned int hardware_concurrency() noexcept {return 1;}
void detach()
{
if (!joinable())
throw system_error();
mHandle = _STD_THREAD_INVALID_HANDLE;
mThreadId.clear();
}
};
namespace this_thread
{
inline thread::id get_id() {return thread::id(GetCurrentThreadId());}
inline void yield() {Sleep(0);}
template< class Rep, class Period >
void sleep_for( const std::chrono::duration<Rep,Period>& sleep_duration)
{
Sleep(chrono::duration_cast<chrono::milliseconds>(sleep_duration).count());
}
template <class Clock, class Duration>
void sleep_until(const std::chrono::time_point<Clock,Duration>& sleep_time)
{
sleep_for(sleep_time-Clock::now());
}
}
}
#endif
#endif

239
tools/mcufontencoder/src/datafile.cc

@ -0,0 +1,239 @@
#include "datafile.hh"
#include <sstream>
#include <algorithm>
#include <cctype>
#include <stdexcept>
#include "ccfixes.hh"
#define DATAFILE_FORMAT_VERSION 1
namespace mcufont {
DataFile::DataFile(const std::vector<dictentry_t> &dictionary,
const std::vector<glyphentry_t> &glyphs,
const fontinfo_t &fontinfo):
m_dictionary(dictionary), m_glyphtable(glyphs), m_fontinfo(fontinfo)
{
dictentry_t dummy = {};
while (m_dictionary.size() < dictionarysize)
m_dictionary.push_back(dummy);
UpdateLowScoreIndex();
}
void DataFile::Save(std::ostream &file) const
{
file << "Version " << DATAFILE_FORMAT_VERSION << std::endl;
file << "FontName " << m_fontinfo.name << std::endl;
file << "MaxWidth " << m_fontinfo.max_width << std::endl;
file << "MaxHeight " << m_fontinfo.max_height << std::endl;
file << "BaselineX " << m_fontinfo.baseline_x << std::endl;
file << "BaselineY " << m_fontinfo.baseline_y << std::endl;
file << "LineHeight " << m_fontinfo.line_height << std::endl;
file << "Flags " << m_fontinfo.flags << std::endl;
file << "RandomSeed " << m_seed << std::endl;
for (const dictentry_t &d : m_dictionary)
{
if (d.replacement.size() != 0)
{
file << "DictEntry " << d.score << " ";
file << d.ref_encode << " " << d.replacement << std::endl;
}
}
for (const glyphentry_t &g : m_glyphtable)
{
file << "Glyph ";
for (size_t i = 0; i < g.chars.size(); i++)
{
if (i != 0) file << ',';
file << g.chars.at(i);
}
file << " " << g.width << " " << g.data << std::endl;
}
}
std::unique_ptr<DataFile> DataFile::Load(std::istream &file)
{
fontinfo_t fontinfo = {};
std::vector<dictentry_t> dictionary;
std::vector<glyphentry_t> glyphtable;
uint32_t seed = 1234;
int version = -1;
std::string line;
while (std::getline(file, line))
{
std::istringstream input(line);
std::string tag;
input >> tag;
if (tag == "Version")
{
input >> version;
}
else if (tag == "FontName")
{
while (std::isspace(input.peek())) input.get();
std::getline(input, fontinfo.name);
}
else if (tag == "MaxWidth")
{
input >> fontinfo.max_width;
}
else if (tag == "MaxHeight")
{
input >> fontinfo.max_height;
}
else if (tag == "BaselineX")
{
input >> fontinfo.baseline_x;
}
else if (tag == "BaselineY")
{
input >> fontinfo.baseline_y;
}
else if (tag == "LineHeight")
{
input >> fontinfo.line_height;
}
else if (tag == "RandomSeed")
{
input >> seed;
}
else if (tag == "Flags")
{
input >> fontinfo.flags;
}
else if (tag == "DictEntry" && dictionary.size() < dictionarysize)
{
dictentry_t d = {};
input >> d.score >> d.ref_encode >> d.replacement;
dictionary.push_back(d);
}
else if (tag == "Glyph")
{
glyphentry_t g = {};
std::string chars;
input >> chars >> g.width >> g.data;
if ((int)g.data.size() != fontinfo.max_width * fontinfo.max_height)
throw std::runtime_error("wrong glyph data length: " + std::to_string(g.data.size()));
size_t pos = 0;
while (pos < chars.size()) {
size_t p;
g.chars.push_back(std::stoi(chars.substr(pos), &p));
pos += p + 1;
}
glyphtable.push_back(g);
}
}
if (version != DATAFILE_FORMAT_VERSION)
{
return std::unique_ptr<DataFile>(nullptr);
}
std::unique_ptr<DataFile> result(new DataFile(dictionary, glyphtable, fontinfo));
result->SetSeed(seed);
return result;
}
void DataFile::SetDictionaryEntry(size_t index, const dictentry_t &value)
{
m_dictionary.at(index) = value;
if (index == m_lowscoreindex ||
m_dictionary.at(m_lowscoreindex).score > value.score)
{
UpdateLowScoreIndex();
}
}
std::map<size_t, size_t> DataFile::GetCharToGlyphMap() const
{
std::map<size_t, size_t> char_to_glyph;
for (size_t i = 0; i < m_glyphtable.size(); i++)
{
for (size_t c: m_glyphtable[i].chars)
{
char_to_glyph[c] = i;
}
}
return char_to_glyph;
}
std::string DataFile::GlyphToText(size_t index) const
{
std::ostringstream os;
const char glyphchars[] = "....,,,,----XXXX";
for (int y = 0; y < m_fontinfo.max_height; y++)
{
for (int x = 0; x < m_fontinfo.max_width; x++)
{
size_t pos = y * m_fontinfo.max_width + x;
os << glyphchars[m_glyphtable.at(index).data.at(pos)];
}
os << std::endl;
}
return os.str();
}
void DataFile::UpdateLowScoreIndex()
{
auto comparison = [](const dictentry_t &a, const dictentry_t &b)
{
return a.score < b.score;
};
auto iter = std::min_element(m_dictionary.begin(),
m_dictionary.end(),
comparison);
m_lowscoreindex = iter - m_dictionary.begin();
}
std::ostream& operator<<(std::ostream& os, const DataFile::pixels_t& str)
{
for (uint8_t p: str)
{
if (p <= 9)
os << (char)(p + '0');
else if (p <= 15)
os << (char)(p - 10 + 'A');
else
throw std::logic_error("invalid pixel alpha: " + std::to_string(p));
}
return os;
}
std::istream& operator>>(std::istream& is, DataFile::pixels_t& str)
{
char c;
str.clear();
while (isspace(is.peek())) is.get();
while (is.get(c))
{
if (c >= '0' && c <= '9')
str.push_back(c - '0');
else if (c >= 'A' && c <= 'F')
str.push_back(c - 'A' + 10);
else
break;
}
return is;
}
}

174
tools/mcufontencoder/src/datafile.hh

@ -0,0 +1,174 @@
// Class to store the data of a font while it is being processed.
// This class can be safely cloned using the default copy constructor.
#pragma once
#include <cstdint>
#include <vector>
#include <string>
#include <fstream>
#include <memory>
#include <map>
namespace mcufont
{
class DataFile
{
public:
typedef std::vector<uint8_t> pixels_t;
struct dictentry_t
{
pixels_t replacement; // The expanded version of this block.
int score; // Number of bytes that having this entry saves.
bool ref_encode; // Encode using references to other dictionary entries.
dictentry_t(): score(0), ref_encode(false) {}
};
struct glyphentry_t
{
pixels_t data; // The full data of the glyph.
std::vector<int> chars; // Characters that this glyph represents.
int width; // Tracking width of the character.
};
struct fontinfo_t
{
std::string name; // Name of the typeface
int max_width; // Width of the character bounding box.
int max_height; // Height of the character bounding box.
int baseline_x; // X coordinate (from left) of the baseline.
int baseline_y; // Y coordinate (from top) of the baseline.
int line_height; // Line height (vertical advance).
int flags;
};
static const int FLAG_MONOSPACE = 0x01;
static const int FLAG_BW = 0x02;
// Construct from data in memory.
DataFile(const std::vector<dictentry_t> &dictionary,
const std::vector<glyphentry_t> &glyphs,
const fontinfo_t &fontinfo);
// Save to a file (custom format)
void Save(std::ostream &file) const;
// Load from a file (custom format)
// Returns nullptr if load fails.
static std::unique_ptr<DataFile> Load(std::istream &file);
// Get or set an entry in the dictionary. The size of the dictionary
// is constant. Entries 0 to 23 are reserved for special purposes.
static const size_t dictionarysize = 256 - 24;
const dictentry_t &GetDictionaryEntry(size_t index) const
{ return m_dictionary.at(index); }
void SetDictionaryEntry(size_t index, const dictentry_t &value);
const std::vector<dictentry_t> &GetDictionary() const
{ return m_dictionary; }
// Get the index of the dictionary entry with the lowest score.
size_t GetLowScoreIndex() const
{ return m_lowscoreindex; }
// Get an entry in the glyph table.
size_t GetGlyphCount() const
{ return m_glyphtable.size(); }
const glyphentry_t &GetGlyphEntry(size_t index) const
{ return m_glyphtable.at(index); }
const std::vector<glyphentry_t> &GetGlyphTable() const
{ return m_glyphtable; }
// Create a map of char indices to glyph indices
std::map<size_t, size_t> GetCharToGlyphMap() const;
// Get the information that applies to all glyphs.
const fontinfo_t &GetFontInfo() const
{ return m_fontinfo; }
// Show a glyph as text.
std::string GlyphToText(size_t index) const;
// Get the random generator seed
// The seed is stored in the datafile to get deterministic behaviour
// for debugging.
uint32_t GetSeed() const { return m_seed; }
void SetSeed(uint32_t seed) { m_seed = seed; }
private:
std::vector<dictentry_t> m_dictionary;
std::vector<glyphentry_t> m_glyphtable;
fontinfo_t m_fontinfo;
uint32_t m_seed;
size_t m_lowscoreindex;
void UpdateLowScoreIndex();
};
std::ostream& operator<<(std::ostream& os, const DataFile::pixels_t& str);
std::istream& operator>>(std::istream& is, DataFile::pixels_t& str);
}
#ifdef CXXTEST_RUNNING
#include <cxxtest/TestSuite.h>
using namespace mcufont;
class DataFileTests: public CxxTest::TestSuite
{
public:
void testFileLoad()
{
std::istringstream s(testfile);
std::unique_ptr<DataFile> f = DataFile::Load(s);
TS_ASSERT_EQUALS(f->GetFontInfo().name, "Sans Serif");
TS_ASSERT_EQUALS(f->GetFontInfo().max_width, 4);
TS_ASSERT_EQUALS(f->GetFontInfo().max_height, 6);
TS_ASSERT_EQUALS(f->GetDictionaryEntry(0).score, 5);
TS_ASSERT_EQUALS(f->GetDictionaryEntry(1).score, 13);
TS_ASSERT_EQUALS(f->GetGlyphCount(), 3);
DataFile::pixels_t expected = {
0,15,0,15, 0,15,0,15, 0,15,0,15, 0,15,0,15, 0,15,0,15, 0,15,0,15
};
TS_ASSERT_EQUALS(f->GetGlyphEntry(0).data.size(), 24);
TS_ASSERT(f->GetGlyphEntry(0).data == expected);
}
void testFileSave()
{
std::istringstream is1(testfile);
std::unique_ptr<DataFile> f1 = DataFile::Load(is1);
std::ostringstream os;
f1->Save(os);
std::string text = os.str();
std::istringstream is2(text);
std::unique_ptr<DataFile> f2 = DataFile::Load(is2);
TS_ASSERT_EQUALS(f1->GetFontInfo().name, f2->GetFontInfo().name);
TS_ASSERT(f1->GetGlyphEntry(0).data == f2->GetGlyphEntry(0).data);
}
private:
static constexpr const char *testfile =
"Version 1\n"
"FontName Sans Serif\n"
"MaxWidth 4\n"
"MaxHeight 6\n"
"BaselineX 1\n"
"BaselineY 1\n"
"DictEntry 5 0 0F0F0\n"
"DictEntry 13 0 F0F0F0\n"
"DictEntry 1 0 F0F0F0\n"
"Glyph 1,2,3 4 0F0F0F0F0F0F0F0F0F0F0F0F\n"
"Glyph 4 4 0F0F0F0F0F0F0F0F0F0F0F0F\n"
"Glyph 5 4 0F0F0F0F0F0F0F0F0F0F0F0F\n";
};
#endif

735
tools/mcufontencoder/src/encode_rlefont.cc

@ -0,0 +1,735 @@
#include "encode_rlefont.hh"
#include <algorithm>
#include <stdexcept>
#include "ccfixes.hh"
// Number of reserved codes before the dictionary entries.
#define DICT_START 24
// Special reference to mean "fill with zeros to the end of the glyph"
#define REF_FILLZEROS 16
// RLE codes
#define RLE_CODEMASK 0xC0
#define RLE_VALMASK 0x3F
#define RLE_ZEROS 0x00 // 0 to 63 zeros
#define RLE_64ZEROS 0x40 // (1 to 64) * 64 zeros
#define RLE_ONES 0x80 // 1 to 64 full alphas
#define RLE_SHADE 0xC0 // 1 to 4 partial alphas
// Dictionary "fill entries" for encoding bits directly.
#define DICT_START7BIT 4
#define DICT_START6BIT 132
#define DICT_START5BIT 196
#define DICT_START4BIT 228
#define DICT_START3BIT 244
#define DICT_START2BIT 252
namespace mcufont {
namespace rlefont {
// Get bit count for the "fill entries"
static size_t fillentry_bitcount(size_t index)
{
if (index >= DICT_START2BIT)
return 2;
else if (index >= DICT_START3BIT)
return 3;
else if (index >= DICT_START4BIT)
return 4;
else if (index >= DICT_START5BIT)
return 5;
else if (index >= DICT_START6BIT)
return 6;
else
return 7;
}
// Count the number of equal pixels at the beginning of the pixelstring.
static size_t prefix_length(const DataFile::pixels_t &pixels, size_t pos)
{
uint8_t pixel = pixels.at(pos);
size_t count = 1;
while (pos + count < pixels.size() &&
pixels.at(pos + count) == pixel)
{
count++;
}
return count;
}
// Perform the RLE encoding for a dictionary entry.
static encoded_font_t::rlestring_t encode_rle(const DataFile::pixels_t &pixels)
{
encoded_font_t::rlestring_t result;
size_t pos = 0;
while (pos < pixels.size())
{
uint8_t pixel = pixels.at(pos);
size_t count = prefix_length(pixels, pos);
pos += count;
if (pixel == 0)
{
// Up to 63 zeros can be encoded with RLE_ZEROS. If there are more,
// encode using RLE_64ZEROS, and then whatever remains with RLE_ZEROS.
while (count >= 64)
{
size_t c = (count > 4096) ? 64 : (count / 64);
result.push_back(RLE_64ZEROS | (c - 1));
count -= c * 64;
}
if (count)
{
result.push_back(RLE_ZEROS | count);
}
}
else if (pixel == 15)
{
// Encode ones.
while (count)
{
size_t c = (count > 64) ? 64 : count;
result.push_back(RLE_ONES | (c - 1));
count -= c;
}
}
else
{
// Encode shades.
while (count)
{
size_t c = (count > 4) ? 4 : count;
result.push_back(RLE_SHADE | ((c - 1) << 4) | pixel);
count -= c;
}
}
}
return result;
}
// We use a tree structure to represent the dictionary entries.
// Using this tree, we can perform a combined Aho-Corasick string matching
// and breadth-first search to find the optimal encoding of glyph data.
class DictTreeNode
{
public:
constexpr DictTreeNode():
m_index(-1),
m_ref(false),
m_length(0),
m_child0(nullptr),
m_child15(nullptr),
m_suffix(nullptr)
{}
void SetChild(uint8_t p, DictTreeNode *child)
{
if (p == 0)
m_child0 = child;
else if (p == 15)
m_child15 = child;
else if (p > 15)
throw std::logic_error("invalid pixel alpha: " + std::to_string(p));
else
{
if (!m_children)
{
m_children.reset(new DictTreeNode*[14]());
}
m_children[p - 1] = child;
}
}
DictTreeNode* GetChild(uint8_t p) const
{
if (p == 0)
return m_child0;
else if (p == 15)
return m_child15;
else if (p > 15)
throw std::logic_error("invalid pixel alpha: " + std::to_string(p));
else if (!m_children)
return nullptr;
else
return m_children[p - 1];
}
bool HasIntermediateChildren() const { return m_children != nullptr; }
int GetIndex() const { return m_index; }
void SetIndex(int index) { m_index = index; }
bool GetRef() const { return m_ref; }
void SetRef(bool ref) { m_ref = ref; }
size_t GetLength() const { return m_length; }
void SetLength(size_t length) { m_length = length; }
DictTreeNode *GetSuffix() const { return m_suffix; }
void SetSuffix(DictTreeNode *suffix) { m_suffix = suffix; }
private:
// Index of dictionary entry or -1 if just a intermediate node.
int m_index;
// True for ref-encoded dictionary entries. Used to avoid recursion when
// encoding them.
bool m_ref;
// Length of the corresponding dictionary entry replacement.
// Equals the distance from the tree root.
size_t m_length;
// Most tree nodes will only ever contains children for 0 or 15.
// Therefore the array for other nodes is allocated only on demand.
DictTreeNode *m_child0;
DictTreeNode *m_child15;
std::unique_ptr<DictTreeNode*[]> m_children;
// Pointer to the longest suffix of this entry that exists in the
// dictionary.
DictTreeNode *m_suffix;
};
// Preallocated array for tree nodes
class TreeAllocator
{
public:
TreeAllocator(size_t count)
{
m_storage.reset(new DictTreeNode[count]);
m_next = m_storage.get();
m_left = count;
}
DictTreeNode *allocate()
{
if (m_left == 0)
throw std::logic_error("Ran out of preallocated entries");
m_left--;
return m_next++;
}
private:
std::unique_ptr<DictTreeNode[]> m_storage;
DictTreeNode *m_next;
size_t m_left;
};
// Add a new dictionary entry to the tree. Adds the intermediate nodes, but
// does not yet fill the suffix pointers.
static DictTreeNode* add_tree_entry(const DataFile::pixels_t &entry, int index,
bool ref_encoded, DictTreeNode *root,
TreeAllocator &storage)
{
DictTreeNode* node = root;
for (uint8_t p : entry)
{
DictTreeNode* branch = node->GetChild(p);
if (!branch)
{
branch = storage.allocate();
node->SetChild(p, branch);
}
node = branch;
}
// Replace the entry if it either does not yet have an encoding, or if
// the new entry is non-ref (i.e. can be used in more situations).
if (node->GetIndex() < 0 || (node->GetRef() && !ref_encoded))
{
node->SetIndex(index);
node->SetRef(ref_encoded);
node->SetLength(entry.size());
}
return node;
}
// Walk the tree and find if the entry exists in the tree. If it does,
// returns a pointer to it, otherwise nullptr.
static DictTreeNode *find_tree_node(DataFile::pixels_t::const_iterator begin,
DataFile::pixels_t::const_iterator end,
DictTreeNode *root)
{
DictTreeNode* node = root;
while (begin != end)
{
uint8_t pixel = *begin++;
node = node->GetChild(pixel);
if (!node)
return nullptr;
}
return node;
}
// Fill in the suffix pointers recursively for the given subtree.
static void fill_tree_suffixes(DictTreeNode *root, DictTreeNode *subtree,
const DataFile::pixels_t &entry)
{
for (size_t i = 1; i < entry.size(); i++)
{
DictTreeNode *node = find_tree_node(entry.begin() + i, entry.end(), root);
if (node)
{
subtree->SetSuffix(node);
break;
}
}
if (!subtree->GetSuffix())
subtree->SetSuffix(root);
DataFile::pixels_t newentry(entry);
newentry.resize(entry.size() + 1);
for (uint8_t i = 0; i < 16; i++)
{
// Speed-up for the common case of 0 and 15 alphas.
if (i == 1 && !subtree->HasIntermediateChildren())
i += 14;
DictTreeNode *child = subtree->GetChild(i);
if (child)
{
newentry.at(entry.size()) = i;
fill_tree_suffixes(root, child, newentry);
}
}
}
// Construct a lookup tree from the dictionary entries.
static DictTreeNode* construct_tree(const std::vector<DataFile::dictentry_t> &dictionary,
TreeAllocator &storage, bool fast)
{
DictTreeNode* root = storage.allocate();
// Populate the hardcoded entries for 0 to 15 alpha.
for (int j = 0; j < 16; j++)
{
DictTreeNode *node = storage.allocate();
node->SetIndex(j);
node->SetRef(false);
node->SetLength(1);
root->SetChild(j, node);
}
// Populate the actual dictionary entries
size_t i = DICT_START;
for (DataFile::dictentry_t d : dictionary)
{
if (!d.replacement.size())
break;
add_tree_entry(d.replacement, i, d.ref_encode, root, storage);
i++;
}
if (!fast)
{
// Populate the fill entries for rest of dictionary
for (; i < 256; i++)
{
DataFile::pixels_t pixels;
size_t bitcount = fillentry_bitcount(i);
uint8_t byte = i - DICT_START7BIT;
for (size_t j = 0; j < bitcount; j++)
{
uint8_t p = (byte & (1 << j)) ? 15 : 0;
pixels.push_back(p);
}
add_tree_entry(pixels, i, false, root, storage);
}
// Fill in the suffix pointers for optimal encoding
DataFile::pixels_t nullentry;
fill_tree_suffixes(root, root, nullentry);
}
return root;
}
// Structure for keeping track of the shortest encoding to reach particular
// point of the pixel string.
struct encoding_link_t
{
// Index of the position prior to the last dictionary entry.
size_t previous;
// Index of the dictionary entry that brings us to this point.
int index;
// Number of links to get here from the start of the string.
size_t length;
constexpr encoding_link_t(): previous(0), index(-1), length(9999999) {}
};
// Perform the reference encoding for a glyph entry (optimal version).
// Uses a modified Aho-Corasick algorithm combined with breadth first search
// to find the shortest representation.
static encoded_font_t::refstring_t encode_ref_slow(const DataFile::pixels_t &pixels,
const DictTreeNode *root,
bool is_glyph)
{
// Chain of encodings. Each entry in this array corresponds to a position
// in the pixel string.
std::unique_ptr<encoding_link_t[]> chain(new encoding_link_t[pixels.size() + 1]);
chain[0].previous = 0;
chain[0].index = 0;
chain[0].length = 0;
// Read the pixels one-by-one and update the encoding links accordingly.
const DictTreeNode *node = root;
for (size_t pos = 0; pos < pixels.size(); pos++)
{
uint8_t pixel = pixels.at(pos);
const DictTreeNode *branch = node->GetChild(pixel);
while (!branch)
{
// Cannot expand this sequence, defer to suffix.
node = node->GetSuffix();
branch = node->GetChild(pixel);
}
node = branch;
// We have arrived at a new node, add it and any proper suffixes to
// the link chain.
const DictTreeNode *suffix = node;
while (suffix != root)
{
if (suffix->GetIndex() >= 0 && (is_glyph || !suffix->GetRef()))
{
encoding_link_t link;
link.previous = pos + 1 - suffix->GetLength();
link.index = suffix->GetIndex();
link.length = chain[link.previous].length + 1;
if (link.length < chain[pos + 1].length)
chain[pos + 1] = link;
}
suffix = suffix->GetSuffix();
}
}
// Check if we can shorten the final encoding using REF_FILLZEROS.
if (is_glyph)
{
for (size_t pos = pixels.size() - 1; pos > 0; pos--)
{
if (pixels.at(pos) != 0)
break;
encoding_link_t link;
link.previous = pos;
link.index = REF_FILLZEROS;
link.length = chain[pos].length + 1;
if (link.length <= chain[pixels.size()].length)
chain[pixels.size()] = link;
}
}
// Backtrack from the final link back to the start and construct the
// encoded string.
encoded_font_t::refstring_t result;
size_t len = chain[pixels.size()].length;
result.resize(len);
size_t pos = pixels.size();
for (size_t i = len; i > 0; i--)
{
result.at(i - 1) = chain[pos].index;
pos = chain[pos].previous;
}
return result;
}
// Walk the tree as far as possible following the given pixel string iterator.
// Returns number of pixels encoded, and index is set to the dictionary reference.
static size_t walk_tree(const DictTreeNode *tree,
DataFile::pixels_t::const_iterator pixels,
DataFile::pixels_t::const_iterator pixelsend,
int &index, bool is_glyph)
{
size_t best_length = 0;
size_t length = 0;
index = -1;
const DictTreeNode* node = tree;
while (pixels != pixelsend)
{
uint8_t pixel = *pixels++;
node = node->GetChild(pixel);
if (!node)
break;
length++;
if (is_glyph || !node->GetRef())
{
if (node->GetIndex() >= 0)
{
index = node->GetIndex();
best_length = length;
}
}
}
if (index < 0)
throw std::logic_error("walk_tree failed to find a valid encoding");
return best_length;
}
// Perform the reference encoding for a glyph entry (fast version).
// Uses a simple greedy search to find select the encodings.
static encoded_font_t::refstring_t encode_ref_fast(const DataFile::pixels_t &pixels,
const DictTreeNode *tree,
bool is_glyph)
{
encoded_font_t::refstring_t result;
// Strip any zeroes from end
size_t end = pixels.size();
if (is_glyph)
{
while (end > 0 && pixels.at(end - 1) == 0) end--;
}
size_t i = 0;
while (i < end)
{
int index;
i += walk_tree(tree, pixels.begin() + i, pixels.end(), index, is_glyph);
result.push_back(index);
}
if (i < pixels.size())
result.push_back(REF_FILLZEROS);
return result;
}
static encoded_font_t::refstring_t encode_ref(const DataFile::pixels_t &pixels,
const DictTreeNode *tree,
bool is_glyph, bool fast)
{
if (fast)
return encode_ref_fast(pixels, tree, is_glyph);
else
return encode_ref_slow(pixels, tree, is_glyph);
}
// Compare dictionary entries by their coding type.
// Sorts RLE-encoded entries first and any empty entries last.
static bool cmp_dict_coding(const DataFile::dictentry_t &a,
const DataFile::dictentry_t &b)
{
if (a.replacement.size() == 0 && b.replacement.size() != 0)
return false;
else if (a.replacement.size() != 0 && b.replacement.size() == 0)
return true;
else if (a.ref_encode == false && b.ref_encode == true)
return true;
else
return false;
}
size_t estimate_tree_node_count(const std::vector<DataFile::dictentry_t> &dict)
{
size_t count = DICT_START; // Preallocated entries
for (const DataFile::dictentry_t &d: dict)
{
count += d.replacement.size();
}
count += 128 * 7; // Fill entries
return count;
}
std::unique_ptr<encoded_font_t> encode_font(const DataFile &datafile,
bool fast)
{
std::unique_ptr<encoded_font_t> result(new encoded_font_t);
// Sort the dictionary so that RLE-coded entries come first.
// This way the two are easy to distinguish based on index.
std::vector<DataFile::dictentry_t> sorted_dict = datafile.GetDictionary();
std::stable_sort(sorted_dict.begin(), sorted_dict.end(), cmp_dict_coding);
// Build the binary tree for looking up references.
size_t count = estimate_tree_node_count(sorted_dict);
TreeAllocator allocator(count);
DictTreeNode* tree = construct_tree(sorted_dict, allocator, fast);
// Encode the dictionary entries, using either RLE or reference method.
for (const DataFile::dictentry_t &d : sorted_dict)
{
if (d.replacement.size() == 0)
{
continue;
}
else if (d.ref_encode)
{
result->ref_dictionary.push_back(encode_ref(d.replacement, tree, false, fast));
}
else
{
result->rle_dictionary.push_back(encode_rle(d.replacement));
}
}
// Then reference-encode the glyphs
for (const DataFile::glyphentry_t &g : datafile.GetGlyphTable())
{
result->glyphs.push_back(encode_ref(g.data, tree, true, fast));
}
// Optionally verify that the encoding was correct.
if (!fast)
{
for (size_t i = 0; i < datafile.GetGlyphCount(); i++)
{
std::unique_ptr<DataFile::pixels_t> decoded =
decode_glyph(*result, i, datafile.GetFontInfo());
if (*decoded != datafile.GetGlyphEntry(i).data)
{
auto iter = std::mismatch(decoded->begin(), decoded->end(),
datafile.GetGlyphEntry(i).data.begin());
size_t pos = iter.first - decoded->begin();
throw std::logic_error("verification of glyph " + std::to_string(i) +
" failed at position " + std::to_string(pos));
}
}
}
return result;
}
size_t get_encoded_size(const encoded_font_t &encoded)
{
size_t total = 0;
for (const encoded_font_t::rlestring_t &r : encoded.rle_dictionary)
{
total += r.size();
if (r.size() != 0)
total += 2; // Offset table entry
}
for (const encoded_font_t::refstring_t &r : encoded.ref_dictionary)
{
total += r.size();
if (r.size() != 0)
total += 2; // Offset table entry
}
for (const encoded_font_t::refstring_t &r : encoded.glyphs)
{
total += r.size();
total += 2; // Offset table entry
total += 1; // Width table entry
}
return total;
}
std::unique_ptr<DataFile::pixels_t> decode_glyph(
const encoded_font_t &encoded,
const encoded_font_t::refstring_t &refstring,
const DataFile::fontinfo_t &fontinfo)
{
std::unique_ptr<DataFile::pixels_t> result(new DataFile::pixels_t);
for (uint8_t ref : refstring)
{
if (ref <= 15)
{
result->push_back(ref);
}
else if (ref == REF_FILLZEROS)
{
result->resize(fontinfo.max_width * fontinfo.max_height, 0);
}
else if (ref < DICT_START)
{
throw std::logic_error("unknown code: " + std::to_string(ref));
}
else if (ref - DICT_START < (int)encoded.rle_dictionary.size())
{
for (uint8_t rle : encoded.rle_dictionary.at(ref - DICT_START))
{
if ((rle & RLE_CODEMASK) == RLE_ZEROS)
{
for (int i = 0; i < (rle & RLE_VALMASK); i++)
{
result->push_back(0);
}
}
else if ((rle & RLE_CODEMASK) == RLE_64ZEROS)
{
for (int i = 0; i < ((rle & RLE_VALMASK) + 1) * 64; i++)
{