Add the mcufont encoder to the tools (including a win32 build) with binaries

ugfx_release_2.6
inmarket 2015-11-22 18:42:11 +10:00
parent 3ae120406e
commit 0ec1a5e4da
29 changed files with 3694 additions and 0 deletions

View File

@ -0,0 +1,2 @@
freetype-2.6.1 source code has not been included in this repository so that you are fully
aware of the license types and restrictions before using it with uGFX.

Binary file not shown.

View File

@ -0,0 +1,43 @@
CXXFLAGS = -O2 -Wall -Werror -Wno-unused-function -Wno-sign-compare -std=c++0x
CXXFLAGS += -ggdb
LDFLAGS += -pthread
# Libfreetype
CXXFLAGS += $(shell freetype-config --cflags)
LDFLAGS += $(shell freetype-config --libs)
# Class to represent font data internally
OBJS = datafile.o
# Utility functions
OBJS += importtools.o exporttools.o
# Import formats
OBJS += bdf_import.o freetype_import.o
# rlefont export format
OBJS += encode_rlefont.o optimize_rlefont.o export_rlefont.o
# bwfont export format
OBJS += export_bwfont.o
all: run_unittests mcufont
clean:
rm -f unittests unittests.cc mcufont $(OBJS)
mcufont: main.o $(OBJS)
g++ $(CXXFLAGS) -o $@ $^ $(LDFLAGS)
unittests.cc: *.hh
cxxtestgen --have-eh --error-printer -o unittests.cc $^
unittests: unittests.cc $(OBJS)
g++ $(CXXFLAGS) -o $@ $^ $(LDFLAGS)
run_unittests: unittests
./unittests
%.o: %.cc *.hh
g++ $(CXXFLAGS) -c $<

View File

@ -0,0 +1,44 @@
ARCH = i686-pc-mingw32-
CXXFLAGS = -O2 -Wall -Werror -Wno-unused-function -Wno-sign-compare -std=c++0x
CXXFLAGS += -ggdb
LDFLAGS += -pthread --static
# Libfreetype
CXXFLAGS += $(shell freetype-config --cflags)
LDFLAGS += $(shell freetype-config --libs)
#FREETYPE2_LIB = ../ugfx/3rdparty/freetype-2.6.1
#CXXFLAGS += -I$(FREETYPE2_LIB)/include
#LDFLAGS += -I$(FREETYPE2_LIB)/lib -lfreetype
# compiler fixes for mingw32
CXXFLAGS += -DNEED_STRING_FIXES -DNEED_THREAD_FIXES
# Class to represent font data internally
OBJS = datafile.o
# Utility functions
OBJS += importtools.o exporttools.o
# Import formats
OBJS += bdf_import.o freetype_import.o
# rlefont export format
OBJS += encode_rlefont.o optimize_rlefont.o export_rlefont.o
# bwfont export format
OBJS += export_bwfont.o
all: mcufont
strip: mcufont.exe
strip mcufont.exe
clean:
rm -f mcufont $(OBJS)
mcufont: main.o $(OBJS)
$(ARCH)g++ $(CXXFLAGS) -o $@ $^ $(LDFLAGS)
%.o: %.cc *.hh
$(ARCH)g++ $(CXXFLAGS) -c $<

View File

@ -0,0 +1,156 @@
#include "bdf_import.hh"
#include "importtools.hh"
#include <sstream>
#include <string>
#include <cctype>
#include <stdexcept>
namespace mcufont {
static std::string toupper(const std::string &input)
{
std::string result;
for (char c: input) result.push_back(::toupper(c));
return result;
}
static int hextoint(char c)
{
if (c >= '0' && c <= '9') return c - '0';
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
throw std::domain_error("Hex digit not in range");
}
static void parse_fontinfo(std::istream &file, DataFile::fontinfo_t &fontinfo)
{
std::string line;
while (std::getline(file, line))
{
std::istringstream s(line);
std::string tag;
s >> tag;
tag = toupper(tag);
if (tag == "FONT")
{
while (isspace(s.peek())) s.get();
std::getline(s, fontinfo.name);
}
else if (tag == "FONTBOUNDINGBOX")
{
int x, y;
s >> fontinfo.max_width >> fontinfo.max_height;
s >> x >> y;
fontinfo.baseline_x = - x;
fontinfo.baseline_y = fontinfo.max_height + y;
}
else if (tag == "STARTCHAR")
{
break;
}
}
}
static bool parse_glyph(std::istream &file, DataFile::glyphentry_t &glyph,
const DataFile::fontinfo_t &fontinfo)
{
glyph.chars.clear();
glyph.width = 0;
// Initialize the character contents to all 0 with proper size.
glyph.data.clear();
glyph.data.resize(fontinfo.max_width * fontinfo.max_height, 0);
int bbx_w = fontinfo.max_width;
int bbx_h = fontinfo.max_height;
int bbx_x = - fontinfo.baseline_x;
int bbx_y = fontinfo.baseline_y - fontinfo.max_height;
// Read glyph metadata
std::string line;
std::string tag;
while (std::getline(file, line))
{
std::istringstream s(line);
s >> tag;
tag = toupper(tag);
if (tag == "ENCODING")
{
int c;
s >> c;
glyph.chars.push_back(c);
}
else if (tag == "DWIDTH")
{
s >> glyph.width;
}
else if (tag == "BBX")
{
s >> bbx_w >> bbx_h >> bbx_x >> bbx_y;
}
else if (tag == "BITMAP")
{
break;
}
}
if (tag != "BITMAP")
return false;
// Read glyph bits
int x0 = fontinfo.baseline_x + bbx_x;
int y = fontinfo.baseline_y - bbx_y - bbx_h;
for (int i = 0; i < bbx_h; i++)
{
std::getline(file, line);
line = toupper(line);
for (int x = 0; x < bbx_w; x++)
{
int nibble = hextoint(line.at(x / 4));
uint8_t pixel = 0;
if (nibble & (8 >> (x % 4)))
pixel = 15;
glyph.data.at(y * fontinfo.max_width + x0 + x) = pixel;
}
y++;
}
std::getline(file, line);
line = toupper(line);
if (line.compare(0, 7, "ENDCHAR") == 0)
return true;
else
return false;
}
std::unique_ptr<DataFile> LoadBDF(std::istream &file)
{
DataFile::fontinfo_t fontinfo = {};
std::vector<DataFile::glyphentry_t> glyphtable;
std::vector<DataFile::dictentry_t> dictionary;
parse_fontinfo(file, fontinfo);
while (file)
{
DataFile::glyphentry_t glyph = {};
if (parse_glyph(file, glyph, fontinfo))
glyphtable.push_back(glyph);
}
eliminate_duplicates(glyphtable);
crop_glyphs(glyphtable, fontinfo);
detect_flags(glyphtable, fontinfo);
fontinfo.line_height = fontinfo.max_height;
std::unique_ptr<DataFile> result(new DataFile(
dictionary, glyphtable, fontinfo));
return result;
}
}

View File

@ -0,0 +1,80 @@
// Function for importing .BDF fonts as data files.
#pragma once
#include "datafile.hh"
namespace mcufont
{
std::unique_ptr<DataFile> LoadBDF(std::istream &file);
}
#ifdef CXXTEST_RUNNING
#include <cxxtest/TestSuite.h>
using namespace mcufont;
class BDFTests: public CxxTest::TestSuite
{
public:
void testLoadBDF()
{
std::istringstream s(testfile);
std::unique_ptr<DataFile> f = LoadBDF(s);
TS_ASSERT_EQUALS(f->GetFontInfo().name, "-Misc-Fixed-Medium-R-Normal--14-130-75-75-C-70-ISO8859-15");
TS_ASSERT_EQUALS(f->GetFontInfo().max_width, 6);
TS_ASSERT_EQUALS(f->GetFontInfo().max_height, 11);
TS_ASSERT_EQUALS(f->GetGlyphCount(), 1);
TS_ASSERT_EQUALS(f->GetGlyphEntry(0).chars.size(), 2);
}
private:
static constexpr const char *testfile =
"STARTFONT 2.1\n"
"FONT -Misc-Fixed-Medium-R-Normal--14-130-75-75-C-70-ISO8859-15\n"
"FONTBOUNDINGBOX 7 14 0 -2\n"
"STARTCHAR defaultchar\n"
"ENCODING 0\n"
"DWIDTH 7 0\n"
"BBX 7 14 0 -2\n"
"BITMAP\n"
"00\n"
"B4\n"
"84\n"
"00\n"
"84\n"
"84\n"
"00\n"
"84\n"
"84\n"
"00\n"
"84\n"
"B4\n"
"00\n"
"00\n"
"ENDCHAR\n"
"STARTCHAR copychar\n"
"ENCODING 2\n"
"DWIDTH 7 0\n"
"BBX 7 14 0 -2\n"
"BITMAP\n"
"00\n"
"B4\n"
"84\n"
"00\n"
"84\n"
"84\n"
"00\n"
"84\n"
"84\n"
"00\n"
"84\n"
"B4\n"
"00\n"
"00\n"
"ENDCHAR\n";
};
#endif

View File

@ -0,0 +1,152 @@
#ifdef NEED_STRING_FIXES
#include <string>
#include <sstream>
#include <stdexcept>
#include <limits>
#include <cstdlib>
namespace std {
template <typename T> inline std::string to_string(T value)
{
std::ostringstream os ;
os << value ;
return os.str() ;
}
inline int stoi( const std::string& str, std::size_t* pos = 0, int base = 10 )
{
const char* begin = str.c_str() ;
char* end = nullptr ;
long value = std::strtol( begin, &end, base ) ;
if( errno == ERANGE || value > std::numeric_limits<int>::max() )
throw std::out_of_range( "stoi: out ofrange" ) ;
if( end == str.c_str() )
throw std::invalid_argument( "stoi: invalid argument" ) ;
if(pos) *pos = end - begin ;
return value ;
}
}
#endif
#ifdef NEED_THREAD_FIXES
#ifndef WIN32STDTHREAD_H
#define WIN32STDTHREAD_H
#include <windows.h>
#include <functional>
#include <memory>
#include <chrono>
#include <system_error>
#include <process.h>
#define _STD_THREAD_INVALID_HANDLE 0
namespace std
{
class thread
{
public:
class id
{
DWORD mId;
void clear() {mId = 0;}
friend class thread;
public:
id(DWORD aId=0):mId(aId){}
bool operator==(const id& other) const {return mId == other.mId;}
};
protected:
HANDLE mHandle;
id mThreadId;
public:
typedef HANDLE native_handle_type;
id get_id() const noexcept {return mThreadId;}
native_handle_type native_handle() const {return mHandle;}
thread(): mHandle(_STD_THREAD_INVALID_HANDLE){}
thread(thread& other)
:mHandle(other.mHandle), mThreadId(other.mThreadId)
{
other.mHandle = _STD_THREAD_INVALID_HANDLE;
other.mThreadId.clear();
}
template<class Function, class... Args>
explicit thread(Function&& f, Args&&... args)
{
typedef decltype(std::bind(f, args...)) Call;
Call* call = new Call(std::bind(f, args...));
mHandle = (HANDLE)_beginthreadex(NULL, 0, threadfunc<Call>,
(LPVOID)call, 0, (unsigned*)&(mThreadId.mId));
}
template <class Call>
static unsigned int __stdcall threadfunc(void* arg)
{
std::unique_ptr<Call> upCall(static_cast<Call*>(arg));
(*upCall)();
return (unsigned long)0;
}
bool joinable() const {return mHandle != _STD_THREAD_INVALID_HANDLE;}
void join()
{
if (get_id() == GetCurrentThreadId())
throw system_error(EDEADLK, generic_category());
if (mHandle == _STD_THREAD_INVALID_HANDLE)
throw system_error(ESRCH, generic_category());
if (!joinable())
throw system_error(EINVAL, generic_category());
WaitForSingleObject(mHandle, INFINITE);
CloseHandle(mHandle);
mHandle = _STD_THREAD_INVALID_HANDLE;
mThreadId.clear();
}
~thread()
{
if (joinable())
std::terminate();
}
thread& operator=(const thread&) = delete;
thread& operator=(thread&& other) noexcept
{
if (joinable())
std::terminate();
swap(std::forward<thread>(other));
return *this;
}
void swap(thread&& other) noexcept
{
std::swap(mHandle, other.mHandle);
std::swap(mThreadId.mId, other.mThreadId.mId);
}
static unsigned int hardware_concurrency() noexcept {return 1;}
void detach()
{
if (!joinable())
throw system_error();
mHandle = _STD_THREAD_INVALID_HANDLE;
mThreadId.clear();
}
};
namespace this_thread
{
inline thread::id get_id() {return thread::id(GetCurrentThreadId());}
inline void yield() {Sleep(0);}
template< class Rep, class Period >
void sleep_for( const std::chrono::duration<Rep,Period>& sleep_duration)
{
Sleep(chrono::duration_cast<chrono::milliseconds>(sleep_duration).count());
}
template <class Clock, class Duration>
void sleep_until(const std::chrono::time_point<Clock,Duration>& sleep_time)
{
sleep_for(sleep_time-Clock::now());
}
}
}
#endif
#endif

View File

@ -0,0 +1,239 @@
#include "datafile.hh"
#include <sstream>
#include <algorithm>
#include <cctype>
#include <stdexcept>
#include "ccfixes.hh"
#define DATAFILE_FORMAT_VERSION 1
namespace mcufont {
DataFile::DataFile(const std::vector<dictentry_t> &dictionary,
const std::vector<glyphentry_t> &glyphs,
const fontinfo_t &fontinfo):
m_dictionary(dictionary), m_glyphtable(glyphs), m_fontinfo(fontinfo)
{
dictentry_t dummy = {};
while (m_dictionary.size() < dictionarysize)
m_dictionary.push_back(dummy);
UpdateLowScoreIndex();
}
void DataFile::Save(std::ostream &file) const
{
file << "Version " << DATAFILE_FORMAT_VERSION << std::endl;
file << "FontName " << m_fontinfo.name << std::endl;
file << "MaxWidth " << m_fontinfo.max_width << std::endl;
file << "MaxHeight " << m_fontinfo.max_height << std::endl;
file << "BaselineX " << m_fontinfo.baseline_x << std::endl;
file << "BaselineY " << m_fontinfo.baseline_y << std::endl;
file << "LineHeight " << m_fontinfo.line_height << std::endl;
file << "Flags " << m_fontinfo.flags << std::endl;
file << "RandomSeed " << m_seed << std::endl;
for (const dictentry_t &d : m_dictionary)
{
if (d.replacement.size() != 0)
{
file << "DictEntry " << d.score << " ";
file << d.ref_encode << " " << d.replacement << std::endl;
}
}
for (const glyphentry_t &g : m_glyphtable)
{
file << "Glyph ";
for (size_t i = 0; i < g.chars.size(); i++)
{
if (i != 0) file << ',';
file << g.chars.at(i);
}
file << " " << g.width << " " << g.data << std::endl;
}
}
std::unique_ptr<DataFile> DataFile::Load(std::istream &file)
{
fontinfo_t fontinfo = {};
std::vector<dictentry_t> dictionary;
std::vector<glyphentry_t> glyphtable;
uint32_t seed = 1234;
int version = -1;
std::string line;
while (std::getline(file, line))
{
std::istringstream input(line);
std::string tag;
input >> tag;
if (tag == "Version")
{
input >> version;
}
else if (tag == "FontName")
{
while (std::isspace(input.peek())) input.get();
std::getline(input, fontinfo.name);
}
else if (tag == "MaxWidth")
{
input >> fontinfo.max_width;
}
else if (tag == "MaxHeight")
{
input >> fontinfo.max_height;
}
else if (tag == "BaselineX")
{
input >> fontinfo.baseline_x;
}
else if (tag == "BaselineY")
{
input >> fontinfo.baseline_y;
}
else if (tag == "LineHeight")
{
input >> fontinfo.line_height;
}
else if (tag == "RandomSeed")
{
input >> seed;
}
else if (tag == "Flags")
{
input >> fontinfo.flags;
}
else if (tag == "DictEntry" && dictionary.size() < dictionarysize)
{
dictentry_t d = {};
input >> d.score >> d.ref_encode >> d.replacement;
dictionary.push_back(d);
}
else if (tag == "Glyph")
{
glyphentry_t g = {};
std::string chars;
input >> chars >> g.width >> g.data;
if ((int)g.data.size() != fontinfo.max_width * fontinfo.max_height)
throw std::runtime_error("wrong glyph data length: " + std::to_string(g.data.size()));
size_t pos = 0;
while (pos < chars.size()) {
size_t p;
g.chars.push_back(std::stoi(chars.substr(pos), &p));
pos += p + 1;
}
glyphtable.push_back(g);
}
}
if (version != DATAFILE_FORMAT_VERSION)
{
return std::unique_ptr<DataFile>(nullptr);
}
std::unique_ptr<DataFile> result(new DataFile(dictionary, glyphtable, fontinfo));
result->SetSeed(seed);
return result;
}
void DataFile::SetDictionaryEntry(size_t index, const dictentry_t &value)
{
m_dictionary.at(index) = value;
if (index == m_lowscoreindex ||
m_dictionary.at(m_lowscoreindex).score > value.score)
{
UpdateLowScoreIndex();
}
}
std::map<size_t, size_t> DataFile::GetCharToGlyphMap() const
{
std::map<size_t, size_t> char_to_glyph;
for (size_t i = 0; i < m_glyphtable.size(); i++)
{
for (size_t c: m_glyphtable[i].chars)
{
char_to_glyph[c] = i;
}
}
return char_to_glyph;
}
std::string DataFile::GlyphToText(size_t index) const
{
std::ostringstream os;
const char glyphchars[] = "....,,,,----XXXX";
for (int y = 0; y < m_fontinfo.max_height; y++)
{
for (int x = 0; x < m_fontinfo.max_width; x++)
{
size_t pos = y * m_fontinfo.max_width + x;
os << glyphchars[m_glyphtable.at(index).data.at(pos)];
}
os << std::endl;
}
return os.str();
}
void DataFile::UpdateLowScoreIndex()
{
auto comparison = [](const dictentry_t &a, const dictentry_t &b)
{
return a.score < b.score;
};
auto iter = std::min_element(m_dictionary.begin(),
m_dictionary.end(),
comparison);
m_lowscoreindex = iter - m_dictionary.begin();
}
std::ostream& operator<<(std::ostream& os, const DataFile::pixels_t& str)
{
for (uint8_t p: str)
{
if (p <= 9)
os << (char)(p + '0');
else if (p <= 15)
os << (char)(p - 10 + 'A');
else
throw std::logic_error("invalid pixel alpha: " + std::to_string(p));
}
return os;
}
std::istream& operator>>(std::istream& is, DataFile::pixels_t& str)
{
char c;
str.clear();
while (isspace(is.peek())) is.get();
while (is.get(c))
{
if (c >= '0' && c <= '9')
str.push_back(c - '0');
else if (c >= 'A' && c <= 'F')
str.push_back(c - 'A' + 10);
else
break;
}
return is;
}
}

View File

@ -0,0 +1,174 @@
// Class to store the data of a font while it is being processed.
// This class can be safely cloned using the default copy constructor.
#pragma once
#include <cstdint>
#include <vector>
#include <string>
#include <fstream>
#include <memory>
#include <map>
namespace mcufont
{
class DataFile
{
public:
typedef std::vector<uint8_t> pixels_t;
struct dictentry_t
{
pixels_t replacement; // The expanded version of this block.
int score; // Number of bytes that having this entry saves.
bool ref_encode; // Encode using references to other dictionary entries.
dictentry_t(): score(0), ref_encode(false) {}
};
struct glyphentry_t
{
pixels_t data; // The full data of the glyph.
std::vector<int> chars; // Characters that this glyph represents.
int width; // Tracking width of the character.
};
struct fontinfo_t
{
std::string name; // Name of the typeface
int max_width; // Width of the character bounding box.
int max_height; // Height of the character bounding box.
int baseline_x; // X coordinate (from left) of the baseline.
int baseline_y; // Y coordinate (from top) of the baseline.
int line_height; // Line height (vertical advance).
int flags;
};
static const int FLAG_MONOSPACE = 0x01;
static const int FLAG_BW = 0x02;
// Construct from data in memory.
DataFile(const std::vector<dictentry_t> &dictionary,
const std::vector<glyphentry_t> &glyphs,
const fontinfo_t &fontinfo);
// Save to a file (custom format)
void Save(std::ostream &file) const;
// Load from a file (custom format)
// Returns nullptr if load fails.
static std::unique_ptr<DataFile> Load(std::istream &file);
// Get or set an entry in the dictionary. The size of the dictionary
// is constant. Entries 0 to 23 are reserved for special purposes.
static const size_t dictionarysize = 256 - 24;
const dictentry_t &GetDictionaryEntry(size_t index) const
{ return m_dictionary.at(index); }
void SetDictionaryEntry(size_t index, const dictentry_t &value);
const std::vector<dictentry_t> &GetDictionary() const
{ return m_dictionary; }
// Get the index of the dictionary entry with the lowest score.
size_t GetLowScoreIndex() const
{ return m_lowscoreindex; }
// Get an entry in the glyph table.
size_t GetGlyphCount() const
{ return m_glyphtable.size(); }
const glyphentry_t &GetGlyphEntry(size_t index) const
{ return m_glyphtable.at(index); }
const std::vector<glyphentry_t> &GetGlyphTable() const
{ return m_glyphtable; }
// Create a map of char indices to glyph indices
std::map<size_t, size_t> GetCharToGlyphMap() const;
// Get the information that applies to all glyphs.
const fontinfo_t &GetFontInfo() const
{ return m_fontinfo; }
// Show a glyph as text.
std::string GlyphToText(size_t index) const;
// Get the random generator seed
// The seed is stored in the datafile to get deterministic behaviour
// for debugging.
uint32_t GetSeed() const { return m_seed; }
void SetSeed(uint32_t seed) { m_seed = seed; }
private:
std::vector<dictentry_t> m_dictionary;
std::vector<glyphentry_t> m_glyphtable;
fontinfo_t m_fontinfo;
uint32_t m_seed;
size_t m_lowscoreindex;
void UpdateLowScoreIndex();
};
std::ostream& operator<<(std::ostream& os, const DataFile::pixels_t& str);
std::istream& operator>>(std::istream& is, DataFile::pixels_t& str);
}
#ifdef CXXTEST_RUNNING
#include <cxxtest/TestSuite.h>
using namespace mcufont;
class DataFileTests: public CxxTest::TestSuite
{
public:
void testFileLoad()
{
std::istringstream s(testfile);
std::unique_ptr<DataFile> f = DataFile::Load(s);
TS_ASSERT_EQUALS(f->GetFontInfo().name, "Sans Serif");
TS_ASSERT_EQUALS(f->GetFontInfo().max_width, 4);
TS_ASSERT_EQUALS(f->GetFontInfo().max_height, 6);
TS_ASSERT_EQUALS(f->GetDictionaryEntry(0).score, 5);
TS_ASSERT_EQUALS(f->GetDictionaryEntry(1).score, 13);
TS_ASSERT_EQUALS(f->GetGlyphCount(), 3);
DataFile::pixels_t expected = {
0,15,0,15, 0,15,0,15, 0,15,0,15, 0,15,0,15, 0,15,0,15, 0,15,0,15
};
TS_ASSERT_EQUALS(f->GetGlyphEntry(0).data.size(), 24);
TS_ASSERT(f->GetGlyphEntry(0).data == expected);
}
void testFileSave()
{
std::istringstream is1(testfile);
std::unique_ptr<DataFile> f1 = DataFile::Load(is1);
std::ostringstream os;
f1->Save(os);
std::string text = os.str();
std::istringstream is2(text);
std::unique_ptr<DataFile> f2 = DataFile::Load(is2);
TS_ASSERT_EQUALS(f1->GetFontInfo().name, f2->GetFontInfo().name);
TS_ASSERT(f1->GetGlyphEntry(0).data == f2->GetGlyphEntry(0).data);
}
private:
static constexpr const char *testfile =
"Version 1\n"
"FontName Sans Serif\n"
"MaxWidth 4\n"
"MaxHeight 6\n"
"BaselineX 1\n"
"BaselineY 1\n"
"DictEntry 5 0 0F0F0\n"
"DictEntry 13 0 F0F0F0\n"
"DictEntry 1 0 F0F0F0\n"
"Glyph 1,2,3 4 0F0F0F0F0F0F0F0F0F0F0F0F\n"
"Glyph 4 4 0F0F0F0F0F0F0F0F0F0F0F0F\n"
"Glyph 5 4 0F0F0F0F0F0F0F0F0F0F0F0F\n";
};
#endif

View File

@ -0,0 +1,735 @@
#include "encode_rlefont.hh"
#include <algorithm>
#include <stdexcept>
#include "ccfixes.hh"
// Number of reserved codes before the dictionary entries.
#define DICT_START 24
// Special reference to mean "fill with zeros to the end of the glyph"
#define REF_FILLZEROS 16
// RLE codes
#define RLE_CODEMASK 0xC0
#define RLE_VALMASK 0x3F
#define RLE_ZEROS 0x00 // 0 to 63 zeros
#define RLE_64ZEROS 0x40 // (1 to 64) * 64 zeros
#define RLE_ONES 0x80 // 1 to 64 full alphas
#define RLE_SHADE 0xC0 // 1 to 4 partial alphas
// Dictionary "fill entries" for encoding bits directly.
#define DICT_START7BIT 4
#define DICT_START6BIT 132
#define DICT_START5BIT 196
#define DICT_START4BIT 228
#define DICT_START3BIT 244
#define DICT_START2BIT 252
namespace mcufont {
namespace rlefont {
// Get bit count for the "fill entries"
static size_t fillentry_bitcount(size_t index)
{
if (index >= DICT_START2BIT)
return 2;
else if (index >= DICT_START3BIT)
return 3;
else if (index >= DICT_START4BIT)
return 4;
else if (index >= DICT_START5BIT)
return 5;
else if (index >= DICT_START6BIT)
return 6;
else
return 7;
}
// Count the number of equal pixels at the beginning of the pixelstring.
static size_t prefix_length(const DataFile::pixels_t &pixels, size_t pos)
{
uint8_t pixel = pixels.at(pos);
size_t count = 1;
while (pos + count < pixels.size() &&
pixels.at(pos + count) == pixel)
{
count++;
}
return count;
}
// Perform the RLE encoding for a dictionary entry.
static encoded_font_t::rlestring_t encode_rle(const DataFile::pixels_t &pixels)
{
encoded_font_t::rlestring_t result;
size_t pos = 0;
while (pos < pixels.size())
{
uint8_t pixel = pixels.at(pos);
size_t count = prefix_length(pixels, pos);
pos += count;
if (pixel == 0)
{
// Up to 63 zeros can be encoded with RLE_ZEROS. If there are more,
// encode using RLE_64ZEROS, and then whatever remains with RLE_ZEROS.
while (count >= 64)
{
size_t c = (count > 4096) ? 64 : (count / 64);
result.push_back(RLE_64ZEROS | (c - 1));
count -= c * 64;
}
if (count)
{
result.push_back(RLE_ZEROS | count);
}
}
else if (pixel == 15)
{
// Encode ones.
while (count)
{
size_t c = (count > 64) ? 64 : count;
result.push_back(RLE_ONES | (c - 1));
count -= c;
}
}
else
{
// Encode shades.
while (count)
{
size_t c = (count > 4) ? 4 : count;
result.push_back(RLE_SHADE | ((c - 1) << 4) | pixel);
count -= c;
}
}
}
return result;
}
// We use a tree structure to represent the dictionary entries.
// Using this tree, we can perform a combined Aho-Corasick string matching
// and breadth-first search to find the optimal encoding of glyph data.
class DictTreeNode
{
public:
constexpr DictTreeNode():
m_index(-1),
m_ref(false),
m_length(0),
m_child0(nullptr),
m_child15(nullptr),
m_suffix(nullptr)
{}
void SetChild(uint8_t p, DictTreeNode *child)
{
if (p == 0)
m_child0 = child;
else if (p == 15)
m_child15 = child;
else if (p > 15)
throw std::logic_error("invalid pixel alpha: " + std::to_string(p));
else
{
if (!m_children)
{
m_children.reset(new DictTreeNode*[14]());
}
m_children[p - 1] = child;
}
}
DictTreeNode* GetChild(uint8_t p) const
{
if (p == 0)
return m_child0;
else if (p == 15)
return m_child15;
else if (p > 15)
throw std::logic_error("invalid pixel alpha: " + std::to_string(p));
else if (!m_children)
return nullptr;
else
return m_children[p - 1];
}
bool HasIntermediateChildren() const { return m_children != nullptr; }
int GetIndex() const { return m_index; }
void SetIndex(int index) { m_index = index; }
bool GetRef() const { return m_ref; }
void SetRef(bool ref) { m_ref = ref; }
size_t GetLength() const { return m_length; }
void SetLength(size_t length) { m_length = length; }
DictTreeNode *GetSuffix() const { return m_suffix; }
void SetSuffix(DictTreeNode *suffix) { m_suffix = suffix; }
private:
// Index of dictionary entry or -1 if just a intermediate node.
int m_index;
// True for ref-encoded dictionary entries. Used to avoid recursion when
// encoding them.
bool m_ref;
// Length of the corresponding dictionary entry replacement.
// Equals the distance from the tree root.
size_t m_length;
// Most tree nodes will only ever contains children for 0 or 15.
// Therefore the array for other nodes is allocated only on demand.
DictTreeNode *m_child0;
DictTreeNode *m_child15;
std::unique_ptr<DictTreeNode*[]> m_children;
// Pointer to the longest suffix of this entry that exists in the
// dictionary.
DictTreeNode *m_suffix;
};
// Preallocated array for tree nodes
class TreeAllocator
{
public:
TreeAllocator(size_t count)
{
m_storage.reset(new DictTreeNode[count]);
m_next = m_storage.get();
m_left = count;
}
DictTreeNode *allocate()
{
if (m_left == 0)
throw std::logic_error("Ran out of preallocated entries");
m_left--;
return m_next++;
}
private:
std::unique_ptr<DictTreeNode[]> m_storage;
DictTreeNode *m_next;
size_t m_left;
};
// Add a new dictionary entry to the tree. Adds the intermediate nodes, but
// does not yet fill the suffix pointers.
static DictTreeNode* add_tree_entry(const DataFile::pixels_t &entry, int index,
bool ref_encoded, DictTreeNode *root,
TreeAllocator &storage)
{
DictTreeNode* node = root;
for (uint8_t p : entry)
{
DictTreeNode* branch = node->GetChild(p);
if (!branch)
{
branch = storage.allocate();
node->SetChild(p, branch);
}
node = branch;
}
// Replace the entry if it either does not yet have an encoding, or if
// the new entry is non-ref (i.e. can be used in more situations).
if (node->GetIndex() < 0 || (node->GetRef() && !ref_encoded))
{
node->SetIndex(index);
node->SetRef(ref_encoded);
node->SetLength(entry.size());
}
return node;
}
// Walk the tree and find if the entry exists in the tree. If it does,
// returns a pointer to it, otherwise nullptr.
static DictTreeNode *find_tree_node(DataFile::pixels_t::const_iterator begin,
DataFile::pixels_t::const_iterator end,
DictTreeNode *root)
{
DictTreeNode* node = root;
while (begin != end)
{
uint8_t pixel = *begin++;
node = node->GetChild(pixel);
if (!node)
return nullptr;
}
return node;
}
// Fill in the suffix pointers recursively for the given subtree.
static void fill_tree_suffixes(DictTreeNode *root, DictTreeNode *subtree,
const DataFile::pixels_t &entry)
{
for (size_t i = 1; i < entry.size(); i++)
{
DictTreeNode *node = find_tree_node(entry.begin() + i, entry.end(), root);
if (node)
{
subtree->SetSuffix(node);
break;
}
}
if (!subtree->GetSuffix())
subtree->SetSuffix(root);
DataFile::pixels_t newentry(entry);
newentry.resize(entry.size() + 1);
for (uint8_t i = 0; i < 16; i++)
{
// Speed-up for the common case of 0 and 15 alphas.
if (i == 1 && !subtree->HasIntermediateChildren())
i += 14;
DictTreeNode *child = subtree->GetChild(i);
if (child)
{
newentry.at(entry.size()) = i;
fill_tree_suffixes(root, child, newentry);
}
}
}
// Construct a lookup tree from the dictionary entries.
static DictTreeNode* construct_tree(const std::vector<DataFile::dictentry_t> &dictionary,
TreeAllocator &storage, bool fast)
{
DictTreeNode* root = storage.allocate();
// Populate the hardcoded entries for 0 to 15 alpha.
for (int j = 0; j < 16; j++)
{
DictTreeNode *node = storage.allocate();
node->SetIndex(j);
node->SetRef(false);
node->SetLength(1);
root->SetChild(j, node);
}
// Populate the actual dictionary entries
size_t i = DICT_START;
for (DataFile::dictentry_t d : dictionary)
{
if (!d.replacement.size())
break;
add_tree_entry(d.replacement, i, d.ref_encode, root, storage);
i++;
}
if (!fast)
{
// Populate the fill entries for rest of dictionary
for (; i < 256; i++)
{
DataFile::pixels_t pixels;
size_t bitcount = fillentry_bitcount(i);
uint8_t byte = i - DICT_START7BIT;
for (size_t j = 0; j < bitcount; j++)
{
uint8_t p = (byte & (1 << j)) ? 15 : 0;
pixels.push_back(p);
}
add_tree_entry(pixels, i, false, root, storage);
}
// Fill in the suffix pointers for optimal encoding
DataFile::pixels_t nullentry;
fill_tree_suffixes(root, root, nullentry);
}
return root;
}
// Structure for keeping track of the shortest encoding to reach particular
// point of the pixel string.
struct encoding_link_t
{
// Index of the position prior to the last dictionary entry.
size_t previous;
// Index of the dictionary entry that brings us to this point.
int index;
// Number of links to get here from the start of the string.
size_t length;
constexpr encoding_link_t(): previous(0), index(-1), length(9999999) {}
};
// Perform the reference encoding for a glyph entry (optimal version).
// Uses a modified Aho-Corasick algorithm combined with breadth first search
// to find the shortest representation.
static encoded_font_t::refstring_t encode_ref_slow(const DataFile::pixels_t &pixels,
const DictTreeNode *root,
bool is_glyph)
{
// Chain of encodings. Each entry in this array corresponds to a position
// in the pixel string.
std::unique_ptr<encoding_link_t[]> chain(new encoding_link_t[pixels.size() + 1]);
chain[0].previous = 0;
chain[0].index = 0;
chain[0].length = 0;
// Read the pixels one-by-one and update the encoding links accordingly.
const DictTreeNode *node = root;
for (size_t pos = 0; pos < pixels.size(); pos++)
{
uint8_t pixel = pixels.at(pos);
const DictTreeNode *branch = node->GetChild(pixel);
while (!branch)
{
// Cannot expand this sequence, defer to suffix.
node = node->GetSuffix();
branch = node->GetChild(pixel);
}
node = branch;
// We have arrived at a new node, add it and any proper suffixes to
// the link chain.
const DictTreeNode *suffix = node;
while (suffix != root)
{
if (suffix->GetIndex() >= 0 && (is_glyph || !suffix->GetRef()))
{
encoding_link_t link;
link.previous = pos + 1 - suffix->GetLength();
link.index = suffix->GetIndex();
link.length = chain[link.previous].length + 1;
if (link.length < chain[pos + 1].length)
chain[pos + 1] = link;
}
suffix = suffix->GetSuffix();
}
}
// Check if we can shorten the final encoding using REF_FILLZEROS.
if (is_glyph)
{
for (size_t pos = pixels.size() - 1; pos > 0; pos--)
{
if (pixels.at(pos) != 0)
break;
encoding_link_t link;
link.previous = pos;
link.index = REF_FILLZEROS;
link.length = chain[pos].length + 1;
if (link.length <= chain[pixels.size()].length)
chain[pixels.size()] = link;
}
}
// Backtrack from the final link back to the start and construct the
// encoded string.
encoded_font_t::refstring_t result;
size_t len = chain[pixels.size()].length;
result.resize(len);
size_t pos = pixels.size();
for (size_t i = len; i > 0; i--)
{
result.at(i - 1) = chain[pos].index;
pos = chain[pos].previous;
}
return result;
}
// Walk the tree as far as possible following the given pixel string iterator.
// Returns number of pixels encoded, and index is set to the dictionary reference.
static size_t walk_tree(const DictTreeNode *tree,
DataFile::pixels_t::const_iterator pixels,
DataFile::pixels_t::const_iterator pixelsend,
int &index, bool is_glyph)
{
size_t best_length = 0;
size_t length = 0;
index = -1;
const DictTreeNode* node = tree;
while (pixels != pixelsend)
{
uint8_t pixel = *pixels++;
node = node->GetChild(pixel);
if (!node)
break;
length++;
if (is_glyph || !node->GetRef())
{
if (node->GetIndex() >= 0)
{
index = node->GetIndex();
best_length = length;
}
}
}
if (index < 0)
throw std::logic_error("walk_tree failed to find a valid encoding");
return best_length;
}
// Perform the reference encoding for a glyph entry (fast version).
// Uses a simple greedy search to find select the encodings.
static encoded_font_t::refstring_t encode_ref_fast(const DataFile::pixels_t &pixels,
const DictTreeNode *tree,
bool is_glyph)
{
encoded_font_t::refstring_t result;
// Strip any zeroes from end
size_t end = pixels.size();
if (is_glyph)
{
while (end > 0 && pixels.at(end - 1) == 0) end--;
}
size_t i = 0;
while (i < end)
{
int index;
i += walk_tree(tree, pixels.begin() + i, pixels.end(), index, is_glyph);
result.push_back(index);
}
if (i < pixels.size())
result.push_back(REF_FILLZEROS);
return result;
}
static encoded_font_t::refstring_t encode_ref(const DataFile::pixels_t &pixels,
const DictTreeNode *tree,
bool is_glyph, bool fast)
{
if (fast)
return encode_ref_fast(pixels, tree, is_glyph);
else
return encode_ref_slow(pixels, tree, is_glyph);
}
// Compare dictionary entries by their coding type.
// Sorts RLE-encoded entries first and any empty entries last.
static bool cmp_dict_coding(const DataFile::dictentry_t &a,
const DataFile::dictentry_t &b)
{
if (a.replacement.size() == 0 && b.replacement.size() != 0)
return false;
else if (a.replacement.size() != 0 && b.replacement.size() == 0)
return true;
else if (a.ref_encode == false && b.ref_encode == true)
return true;
else
return false;
}
size_t estimate_tree_node_count(const std::vector<DataFile::dictentry_t> &dict)
{
size_t count = DICT_START; // Preallocated entries
for (const DataFile::dictentry_t &d: dict)
{
count += d.replacement.size();
}
count += 128 * 7; // Fill entries
return count;
}
std::unique_ptr<encoded_font_t> encode_font(const DataFile &datafile,
bool fast)
{
std::unique_ptr<encoded_font_t> result(new encoded_font_t);
// Sort the dictionary so that RLE-coded entries come first.
// This way the two are easy to distinguish based on index.
std::vector<DataFile::dictentry_t> sorted_dict = datafile.GetDictionary();
std::stable_sort(sorted_dict.begin(), sorted_dict.end(), cmp_dict_coding);
// Build the binary tree for looking up references.
size_t count = estimate_tree_node_count(sorted_dict);
TreeAllocator allocator(count);
DictTreeNode* tree = construct_tree(sorted_dict, allocator, fast);
// Encode the dictionary entries, using either RLE or reference method.
for (const DataFile::dictentry_t &d : sorted_dict)
{
if (d.replacement.size() == 0)
{
continue;
}
else if (d.ref_encode)
{
result->ref_dictionary.push_back(encode_ref(d.replacement, tree, false, fast));
}
else
{
result->rle_dictionary.push_back(encode_rle(d.replacement));
}
}
// Then reference-encode the glyphs
for (const DataFile::glyphentry_t &g : datafile.GetGlyphTable())
{
result->glyphs.push_back(encode_ref(g.data, tree, true, fast));
}
// Optionally verify that the encoding was correct.
if (!fast)
{
for (size_t i = 0; i < datafile.GetGlyphCount(); i++)
{
std::unique_ptr<DataFile::pixels_t> decoded =
decode_glyph(*result, i, datafile.GetFontInfo());
if (*decoded != datafile.GetGlyphEntry(i).data)
{
auto iter = std::mismatch(decoded->begin(), decoded->end(),
datafile.GetGlyphEntry(i).data.begin());
size_t pos = iter.first - decoded->begin();
throw std::logic_error("verification of glyph " + std::to_string(i) +
" failed at position " + std::to_string(pos));
}
}
}
return result;
}
size_t get_encoded_size(const encoded_font_t &encoded)
{
size_t total = 0;
for (const encoded_font_t::rlestring_t &r : encoded.rle_dictionary)
{
total += r.size();
if (r.size() != 0)
total += 2; // Offset table entry
}
for (const encoded_font_t::refstring_t &r : encoded.ref_dictionary)
{
total += r.size();
if (r.size() != 0)
total += 2; // Offset table entry
}
for (const encoded_font_t::refstring_t &r : encoded.glyphs)
{
total += r.size();
total += 2; // Offset table entry
total += 1; // Width table entry
}
return total;
}
std::unique_ptr<DataFile::pixels_t> decode_glyph(
const encoded_font_t &encoded,
const encoded_font_t::refstring_t &refstring,
const DataFile::fontinfo_t &fontinfo)
{
std::unique_ptr<DataFile::pixels_t> result(new DataFile::pixels_t);
for (uint8_t ref : refstring)
{
if (ref <= 15)
{
result->push_back(ref);
}
else if (ref == REF_FILLZEROS)
{
result->resize(fontinfo.max_width * fontinfo.max_height, 0);
}
else if (ref < DICT_START)
{
throw std::logic_error("unknown code: " + std::to_string(ref));
}
else if (ref - DICT_START < (int)encoded.rle_dictionary.size())
{
for (uint8_t rle : encoded.rle_dictionary.at(ref - DICT_START))
{
if ((rle & RLE_CODEMASK) == RLE_ZEROS)
{
for (int i = 0; i < (rle & RLE_VALMASK); i++)
{
result->push_back(0);
}
}
else if ((rle & RLE_CODEMASK) == RLE_64ZEROS)
{
for (int i = 0; i < ((rle & RLE_VALMASK) + 1) * 64; i++)
{
result->push_back(0);
}
}
else if ((rle & RLE_CODEMASK) == RLE_ONES)
{
for (int i = 0; i < (rle & RLE_VALMASK) + 1; i++)
{
result->push_back(15);
}
}
else if ((rle & RLE_CODEMASK) == RLE_SHADE)
{
uint8_t count, alpha;
count = ((rle & RLE_VALMASK) >> 4) + 1;
alpha = ((rle & RLE_VALMASK) & 0xF);
for (int i = 0; i < count; i++)
{
result->push_back(alpha);
}
}
}
}
else if (ref - DICT_START - encoded.rle_dictionary.size() < encoded.ref_dictionary.size())
{
size_t index = ref - DICT_START - encoded.rle_dictionary.size();
std::unique_ptr<DataFile::pixels_t> part =
decode_glyph(encoded, encoded.ref_dictionary.at(index),
fontinfo);
result->insert(result->end(), part->begin(), part->end());
}
else
{
size_t bitcount = fillentry_bitcount(ref);
uint8_t byte = ref - DICT_START7BIT;
for (size_t i = 0; i < bitcount; i++)
{
uint8_t p = (byte & (1 << i)) ? 15 : 0;
result->push_back(p);
}
}
}
return result;
}
std::unique_ptr<DataFile::pixels_t> decode_glyph(
const encoded_font_t &encoded, size_t index,
const DataFile::fontinfo_t &fontinfo)
{
return decode_glyph(encoded, encoded.glyphs.at(index), fontinfo);
}
}}

View File

@ -0,0 +1,126 @@
// Given a dictionary and glyphs, encode the data for all the glyphs.
#pragma once
#include "datafile.hh"
#include <vector>
#include <memory>
namespace mcufont {
namespace rlefont {
struct encoded_font_t
{
// RLE-encoded format for storing the dictionary entries.
// Each item is a byte. Top bit means the value in the original bitstream,
// and the bottom 7 bits store the repetition count.
typedef std::vector<uint8_t> rlestring_t;
// Reference encoded format for storing the glyphs.
// Each item is a reference to the dictionary.
// Values 0 and 1 are hardcoded to mean 0 and 1.
// All other values mean dictionary entry at (i-2).
typedef std::vector<uint8_t> refstring_t;
std::vector<rlestring_t> rle_dictionary;
std::vector<refstring_t> ref_dictionary;
std::vector<refstring_t> glyphs;
};
// Encode all the glyphs.
std::unique_ptr<encoded_font_t> encode_font(const DataFile &datafile,
bool fast = true);
// Sum up the total size of the encoded glyphs + dictionary.
size_t get_encoded_size(const encoded_font_t &encoded);
inline size_t get_encoded_size(const DataFile &datafile, bool fast = true)
{
std::unique_ptr<encoded_font_t> e = encode_font(datafile, fast);
return get_encoded_size(*e);
}
// Decode a single glyph (for verification).
std::unique_ptr<DataFile::pixels_t> decode_glyph(
const encoded_font_t &encoded,
const encoded_font_t::refstring_t &refstring,
const DataFile::fontinfo_t &fontinfo);
// Decode a single glyph (for verification).
std::unique_ptr<DataFile::pixels_t> decode_glyph(
const encoded_font_t &encoded, size_t index,
const DataFile::fontinfo_t &fontinfo);
}}
#ifdef CXXTEST_RUNNING
#include <cxxtest/TestSuite.h>
using namespace mcufont;
using namespace mcufont::rlefont;
class RLEFontEncodeTests: public CxxTest::TestSuite
{
public:
void testEncode()
{
std::istringstream s(testfile);
std::unique_ptr<DataFile> f = DataFile::Load(s);
std::unique_ptr<encoded_font_t> e = encode_font(*f, false);
TS_ASSERT_EQUALS(e->glyphs.size(), 3);
// Expected values for dictionary
encoded_font_t::rlestring_t dict0 = {0x01, 0xCE, 0x01, 0xCE};
encoded_font_t::rlestring_t dict1 = {0x0C};
encoded_font_t::rlestring_t dict2 = {0xFE};
encoded_font_t::refstring_t dict3 = {24, 24};
TS_ASSERT(e->rle_dictionary.at(0) == dict0);
TS_ASSERT(e->rle_dictionary.at(1) == dict1);
TS_ASSERT(e->rle_dictionary.at(2) == dict2);
TS_ASSERT(e->ref_dictionary.at(0) == dict3);
// Expected values for glyphs
encoded_font_t::refstring_t glyph0 = {27, 27, 27};
encoded_font_t::refstring_t glyph1 = {24, 0, 132, 25, 14};
encoded_font_t::refstring_t glyph2 = {228, 26, 244, 14, 14, 14, 228, 26, 16};
TS_ASSERT_EQUALS(e->glyphs.at(0), glyph0);
TS_ASSERT_EQUALS(e->glyphs.at(1), glyph1);
TS_ASSERT_EQUALS(e->glyphs.at(2), glyph2);
}
void testDecode()
{
std::istringstream s(testfile);
std::unique_ptr<DataFile> f = DataFile::Load(s);
std::unique_ptr<encoded_font_t> e = encode_font(*f, false);
for (size_t i = 0; i < 3; i++)
{
std::unique_ptr<DataFile::pixels_t> dec;
dec = decode_glyph(*e, i, f->GetFontInfo());
TS_ASSERT_EQUALS(*dec, f->GetGlyphEntry(i).data);
}
}
private:
static constexpr const char *testfile =
"Version 1\n"
"FontName Sans Serif\n"
"MaxWidth 4\n"
"MaxHeight 6\n"
"BaselineX 1\n"
"BaselineY 1\n"
"DictEntry 1 0 0E0E\n"
"DictEntry 1 0 000000000000\n"
"DictEntry 1 0 EEEE\n"
"DictEntry 1 1 0E0E0E0E\n"
"Glyph 0 4 0E0E0E0E0E0E0E0E0E0E0E0E\n"
"Glyph 1 4 0E0E0000000000000000000E\n"
"Glyph 2 4 0000EEEE000EEE0000EEEE00\n";
};
#endif

View File

@ -0,0 +1,247 @@
#include "export_bwfont.hh"
#include <vector>
#include <iomanip>
#include <map>
#include <set>
#include <algorithm>
#include <string>
#include <cctype>
#include "exporttools.hh"
#include "importtools.hh"
#include "ccfixes.hh"
#define BWFONT_FORMAT_VERSION 4
namespace mcufont {
namespace bwfont {
static void encode_glyph(const DataFile::glyphentry_t &glyph,
const DataFile::fontinfo_t &fontinfo,
std::vector<unsigned> &dest,
int num_cols)
{
const int threshold = 8;
// Find the number of columns in the glyph data
if (num_cols == 0)
{
for (int x = 0; x < fontinfo.max_width; x++)
{
for (int y = 0; y < fontinfo.max_height; y++)
{
size_t index = y * fontinfo.max_width + x;
if (glyph.data.at(index) >= threshold)
num_cols = x + 1;
}
}
}
// Write the bits that compose the glyph
for (int x = 0; x < num_cols; x++)
{
for (int y = 0; y < fontinfo.max_height; y+= 8)
{
size_t remain = std::min(8, fontinfo.max_height - y);
uint8_t byte = 0;
for (size_t i = 0; i < remain; i++)
{
size_t index = (y + i) * fontinfo.max_width + x;
if (glyph.data.at(index) >= threshold)
{
byte |= (1 << i);
}
}
dest.push_back(byte);
}
}
}
struct cropinfo_t
{
size_t offset_x;
size_t offset_y;
size_t height_bytes;
size_t height_pixels;
size_t width;
};
static void encode_character_range(std::ostream &out,
const std::string &name,
const DataFile &datafile,
const char_range_t &range,
unsigned range_index,
cropinfo_t &cropinfo)
{
std::vector<DataFile::glyphentry_t> glyphs;
bool constant_width = true;
int width = datafile.GetGlyphEntry(range.glyph_indices[0]).width;
// Copy all the glyphs in this range for the purpose of cropping them.
for (int glyph_index: range.glyph_indices)
{
if (glyph_index < 0)
{
// Missing glyph
DataFile::glyphentry_t dummy = {};
glyphs.push_back(dummy);
}
else
{
auto glyph = datafile.GetGlyphEntry(glyph_index);
glyphs.push_back(glyph);
if (glyph.width != width)
{
constant_width = false;
width = 0;
}
}
}
// Crop the glyphs in this range. Getting rid of a few rows at top
// or left can save a bunch of bytes with minimal cost.
DataFile::fontinfo_t old_fi = datafile.GetFontInfo();
DataFile::fontinfo_t new_fi = old_fi;
crop_glyphs(glyphs, new_fi);
if (new_fi.max_width != width)
{
constant_width = false;
width = 0;
}
// Fill in the crop information
cropinfo.offset_x = old_fi.baseline_x - new_fi.baseline_x;
cropinfo.offset_y = old_fi.baseline_y - new_fi.baseline_y;
cropinfo.height_pixels = new_fi.max_height;
cropinfo.height_bytes = (cropinfo.height_pixels + 7) / 8;
cropinfo.width = width;
// Then format and write out the glyph data
std::vector<unsigned> offsets;
std::vector<unsigned> data;
std::vector<unsigned> widths;
size_t stride = cropinfo.height_bytes;
for (const DataFile::glyphentry_t &g : glyphs)
{
offsets.push_back(data.size() / stride);
widths.push_back(g.width);
encode_glyph(g, new_fi, data, width);
}
offsets.push_back(data.size() / stride);
write_const_table(out, data, "uint8_t", "mf_bwfont_" + name + "_glyph_data_" + std::to_string(range_index));
if (!constant_width)
{
write_const_table(out, offsets, "uint16_t", "mf_bwfont_" + name + "_glyph_offsets_" + std::to_string(range_index), 4);
write_const_table(out, widths, "uint8_t", "mf_bwfont_" + name + "_glyph_widths_" + std::to_string(range_index));
}
}
void write_source(std::ostream &out, std::string name, const DataFile &datafile)
{
name = filename_to_identifier(name);
out << std::endl;
out << std::endl;
out << "/* Start of automatically generated font definition for " << name << ". */" << std::endl;
out << std::endl;
out << "#ifndef MF_BWFONT_INTERNALS" << std::endl;
out << "#define MF_BWFONT_INTERNALS" << std::endl;
out << "#endif" << std::endl;
out << "#include \"mf_bwfont.h\"" << std::endl;
out << std::endl;
out << "#ifndef MF_BWFONT_VERSION_" << BWFONT_FORMAT_VERSION << "_SUPPORTED" << std::endl;
out << "#error The font file is not compatible with this version of mcufont." << std::endl;
out << "#endif" << std::endl;
out << std::endl;
// Split the characters into ranges
DataFile::fontinfo_t f = datafile.GetFontInfo();
size_t glyph_size = f.max_width * ((f.max_height + 7) / 8);
auto get_glyph_size = [=](size_t i) { return glyph_size; };
std::vector<char_range_t> ranges = compute_char_ranges(datafile,
get_glyph_size, 65536, 16);
// Write out glyph data for character ranges
std::vector<cropinfo_t> crops;
for (size_t i = 0; i < ranges.size(); i++)
{
cropinfo_t cropinfo;
encode_character_range(out, name, datafile, ranges.at(i), i, cropinfo);
crops.push_back(cropinfo);
}
// Write out a table describing the character ranges
out << "static const struct mf_bwfont_char_range_s mf_bwfont_" + name + "_char_ranges[] = {" << std::endl;
for (size_t i = 0; i < ranges.size(); i++)
{
std::string offsets = (crops[i].width) ? "0" : "mf_bwfont_" + name + "_glyph_offsets_" + std::to_string(i);
std::string widths = (crops[i].width) ? "0" : "mf_bwfont_" + name + "_glyph_widths_" + std::to_string(i);
out << " {" << std::endl;
out << " " << ranges.at(i).first_char << ", /* first char */" << std::endl;
out << " " << ranges.at(i).char_count << ", /* char count */" << std::endl;
out << " " << crops[i].offset_x << ", /* offset x */" << std::endl;
out << " " << crops[i].offset_y << ", /* offset y */" << std::endl;
out << " " << crops[i].height_bytes << ", /* height in bytes */" << std::endl;
out << " " << crops[i].height_pixels << ", /* height in pixels */" << std::endl;
out << " " << crops[i].width << ", /* width */" << std::endl;
out << " " << widths << ", /* glyph widths */" << std::endl;
out << " " << offsets << ", /* glyph offsets */" << std::endl;
out << " " << "mf_bwfont_" << name << "_glyph_data_" << i << ", /* glyph data */" << std::endl;
out << " }," << std::endl;
}
out << "};" << std::endl;
out << std::endl;
// Fonts in this format are always black & white
int flags = datafile.GetFontInfo().flags | DataFile::FLAG_BW;
// Pull it all together in the rlefont_s structure.
out << "const struct mf_bwfont_s mf_bwfont_" << name << " = {" << std::endl;
out << " {" << std::endl;
out << " " << "\"" << datafile.GetFontInfo().name << "\"," << std::endl;
out << " " << "\"" << name << "\"," << std::endl;
out << " " << datafile.GetFontInfo().max_width << ", /* width */" << std::endl;
out << " " << datafile.GetFontInfo().max_height << ", /* height */" << std::endl;
out << " " << get_min_x_advance(datafile) << ", /* min x advance */" << std::endl;
out << " " << get_max_x_advance(datafile) << ", /* max x advance */" << std::endl;
out << " " << datafile.GetFontInfo().baseline_x << ", /* baseline x */" << std::endl;
out << " " << datafile.GetFontInfo().baseline_y << ", /* baseline y */" << std::endl;
out << " " << datafile.GetFontInfo().line_height << ", /* line height */" << std::endl;
out << " " << flags << ", /* flags */" << std::endl;
out << " " << select_fallback_char(datafile) << ", /* fallback character */" << std::endl;
out << " " << "&mf_bwfont_character_width," << std::endl;
out << " " << "&mf_bwfont_render_character," << std::endl;
out << " }," << std::endl;
out << " " << BWFONT_FORMAT_VERSION << ", /* version */" << std::endl;
out << " " << ranges.size() << ", /* char range count */" << std::endl;
out << " " << "mf_bwfont_" << name << "_char_ranges," << std::endl;
out << "};" << std::endl;
// Write the font lookup structure
out << std::endl;
out << "#ifdef MF_INCLUDED_FONTS" << std::endl;
out << "/* List entry for searching fonts by name. */" << std::endl;
out << "static const struct mf_font_list_s mf_bwfont_" << name << "_listentry = {" << std::endl;
out << " MF_INCLUDED_FONTS," << std::endl;
out << " (struct mf_font_s*)&mf_bwfont_" << name << std::endl;
out << "};" << std::endl;
out << "#undef MF_INCLUDED_FONTS" << std::endl;
out << "#define MF_INCLUDED_FONTS (&mf_bwfont_" << name << "_listentry)" << std::endl;
out << "#endif" << std::endl;
out << std::endl;
out << std::endl;
out << "/* End of automatically generated font definition for " << name << ". */" << std::endl;
out << std::endl;
}
}}

View File

@ -0,0 +1,16 @@
// Write out the encoded data in C source code files for mf_bwfont format.
#pragma once
#include "datafile.hh"
#include <iostream>
namespace mcufont {
namespace bwfont {
void write_header(std::ostream &out, std::string name, const DataFile &datafile);
void write_source(std::ostream &out, std::string name, const DataFile &datafile);
} }

View File

@ -0,0 +1,181 @@
#include "export_rlefont.hh"
#include <vector>
#include <iomanip>
#include <map>
#include <set>
#include <algorithm>
#include <string>
#include <cctype>
#include "exporttools.hh"
#include "ccfixes.hh"
#define RLEFONT_FORMAT_VERSION 4
namespace mcufont {
namespace rlefont {
// Encode the dictionary entries and the offsets to them.
// Generates tables dictionary_data and dictionary_offsets.
static void encode_dictionary(std::ostream &out,
const std::string &name,
const DataFile &datafile,
const encoded_font_t &encoded)
{
std::vector<unsigned> offsets;
std::vector<unsigned> data;
for (const encoded_font_t::rlestring_t &r : encoded.rle_dictionary)
{
offsets.push_back(data.size());
data.insert(data.end(), r.begin(), r.end());
}
for (const encoded_font_t::refstring_t &r : encoded.ref_dictionary)
{
offsets.push_back(data.size());
data.insert(data.end(), r.begin(), r.end());
}
offsets.push_back(data.size());
write_const_table(out, data, "uint8_t", "mf_rlefont_" + name + "_dictionary_data");
write_const_table(out, offsets, "uint16_t", "mf_rlefont_" + name + "_dictionary_offsets", 4);
}
// Encode the data tables for a single character range.
// Generates tables glyph_data_i and glyph_offsets_i.
static void encode_character_range(std::ostream &out,
const std::string &name,
const DataFile &datafile,
const encoded_font_t& encoded,
const char_range_t& range,
unsigned range_index)
{
std::vector<unsigned> offsets;
std::vector<unsigned> data;
std::map<size_t, unsigned> already_encoded;
for (int glyph_index : range.glyph_indices)
{
if (already_encoded.count(glyph_index))
{
offsets.push_back(already_encoded[glyph_index]);
}
else
{
encoded_font_t::refstring_t r;
int width = 0;
if (glyph_index >= 0)
{
r = encoded.glyphs[glyph_index];
width = datafile.GetGlyphEntry(glyph_index).width;
}
offsets.push_back(data.size());
already_encoded[glyph_index] = data.size();
data.push_back(width);
data.insert(data.end(), r.begin(), r.end());
}
}
write_const_table(out, data, "uint8_t", "mf_rlefont_" + name + "_glyph_data_" + std::to_string(range_index));
write_const_table(out, offsets, "uint16_t", "mf_rlefont_" + name + "_glyph_offsets_" + std::to_string(range_index), 4);
}
void write_source(std::ostream &out, std::string name, const DataFile &datafile)
{
name = filename_to_identifier(name);
std::unique_ptr<encoded_font_t> encoded = encode_font(datafile, false);
out << std::endl;
out << std::endl;
out << "/* Start of automatically generated font definition for " << name << ". */" << std::endl;
out << std::endl;
out << "#ifndef MF_RLEFONT_INTERNALS" << std::endl;
out << "#define MF_RLEFONT_INTERNALS" << std::endl;
out << "#endif" << std::endl;
out << "#include \"mf_rlefont.h\"" << std::endl;
out << std::endl;
out << "#ifndef MF_RLEFONT_VERSION_" << RLEFONT_FORMAT_VERSION << "_SUPPORTED" << std::endl;
out << "#error The font file is not compatible with this version of mcufont." << std::endl;
out << "#endif" << std::endl;
out << std::endl;
// Write out the dictionary entries
encode_dictionary(out, name, datafile, *encoded);
// Split the characters into ranges
auto get_glyph_size = [&encoded](size_t i)
{
return encoded->glyphs[i].size();
};
std::vector<char_range_t> ranges = compute_char_ranges(datafile,
get_glyph_size, 65536, 16);
// Write out glyph data for character ranges
for (size_t i = 0; i < ranges.size(); i++)
{
encode_character_range(out, name, datafile, *encoded, ranges.at(i), i);
}
// Write out a table describing the character ranges
out << "static const struct mf_rlefont_char_range_s mf_rlefont_" << name << "_char_ranges[] = {" << std::endl;
for (size_t i = 0; i < ranges.size(); i++)
{
out << " {" << ranges.at(i).first_char
<< ", " << ranges.at(i).char_count
<< ", mf_rlefont_" << name << "_glyph_offsets_" << i
<< ", mf_rlefont_" << name << "_glyph_data_" << i << "}," << std::endl;
}
out << "};" << std::endl;
out << std::endl;
// Pull it all together in the rlefont_s structure.
out << "const struct mf_rlefont_s mf_rlefont_" << name << " = {" << std::endl;
out << " {" << std::endl;
out << " " << "\"" << datafile.GetFontInfo().name << "\"," << std::endl;
out << " " << "\"" << name << "\"," << std::endl;
out << " " << datafile.GetFontInfo().max_width << ", /* width */" << std::endl;
out << " " << datafile.GetFontInfo().max_height << ", /* height */" << std::endl;
out << " " << get_min_x_advance(datafile) << ", /* min x advance */" << std::endl;
out << " " << get_max_x_advance(datafile) << ", /* max x advance */" << std::endl;
out << " " << datafile.GetFontInfo().baseline_x << ", /* baseline x */" << std::endl;
out << " " << datafile.GetFontInfo().baseline_y << ", /* baseline y */" << std::endl;
out << " " << datafile.GetFontInfo().line_height << ", /* line height */" << std::endl;
out << " " << datafile.GetFontInfo().flags << ", /* flags */" << std::endl;
out << " " << select_fallback_char(datafile) << ", /* fallback character */" << std::endl;
out << " " << "&mf_rlefont_character_width," << std::endl;
out << " " << "&mf_rlefont_render_character," << std::endl;
out << " }," << std::endl;
out << " " << RLEFONT_FORMAT_VERSION << ", /* version */" << std::endl;
out << " " << "mf_rlefont_" << name << "_dictionary_data," << std::endl;
out << " " << "mf_rlefont_" << name << "_dictionary_offsets," << std::endl;
out << " " << encoded->rle_dictionary.size() << ", /* rle dict count */" << std::endl;
out << " " << encoded->ref_dictionary.size() + encoded->rle_dictionary.size() << ", /* total dict count */" << std::endl;
out << " " << ranges.size() << ", /* char range count */" << std::endl;
out << " " << "mf_rlefont_" << name << "_char_ranges," << std::endl;
out << "};" << std::endl;
// Write the font lookup structure
out << std::endl;
out << "#ifdef MF_INCLUDED_FONTS" << std::endl;
out << "/* List entry for searching fonts by name. */" << std::endl;
out << "static const struct mf_font_list_s mf_rlefont_" << name << "_listentry = {" << std::endl;
out << " MF_INCLUDED_FONTS," << std::endl;
out << " (struct mf_font_s*)&mf_rlefont_" << name << std::endl;
out << "};" << std::endl;
out << "#undef MF_INCLUDED_FONTS" << std::endl;
out << "#define MF_INCLUDED_FONTS (&mf_rlefont_" << name << "_listentry)" << std::endl;
out << "#endif" << std::endl;
out << std::endl;
out << std::endl;
out << "/* End of automatically generated font definition for " << name << ". */" << std::endl;
out << std::endl;
}
}}

View File

@ -0,0 +1,15 @@
// Write out the encoded data in C source code files for the mf_rlefont format.
#pragma once
#include "datafile.hh"
#include "encode_rlefont.hh"
#include <iostream>
namespace mcufont {
namespace rlefont {
void write_source(std::ostream &out, std::string name, const DataFile &datafile);
} }

View File

@ -0,0 +1,179 @@
#include "exporttools.hh"
#include <iomanip>
#include <set>
namespace mcufont {
// Convert a file name to a valid C identifier
std::string filename_to_identifier(std::string name)
{
// If the name contains path separators (/ or \), take only the last part.
size_t pos = name.find_last_of("/\\");
if (pos != std::string::npos)
name = name.substr(pos + 1);
// If the name contains a file extension, strip it.
pos = name.find_first_of(".");
if (pos != std::string::npos)
name = name.substr(0, pos);
// Replace any special characters with _.
for (pos = 0; pos < name.size(); pos++)
{
if (!isalnum(name.at(pos)))
name.at(pos) = '_';
}
return name;
}
// Write a vector of integers as line-wrapped hex/integer data for initializing const array.
void wordwrap_vector(std::ostream &out, const std::vector<unsigned> &data,
const std::string &prefix, size_t width)
{
int values_per_column = (width <= 2) ? 16 : 8;
std::ios::fmtflags flags(out.flags());
out << prefix;
out << std::hex << std::setfill('0');
for (size_t i = 0; i < data.size(); i++)
{
if (i % values_per_column == 0 && i != 0)
out << std::endl << prefix;
out << "0x" << std::setw(width) << (int)data.at(i) << ", ";
}
out.flags(flags);
}
// Write a vector of integers as a C constant array of given datatype.
void write_const_table(std::ostream &out, const std::vector<unsigned> &data,
const std::string &datatype, const std::string &tablename,
size_t width)
{
out << "static const " << datatype << " " << tablename;
out << "[" << data.size() << "] = {" << std::endl;
wordwrap_vector(out, data, " ", width);
out << std::endl << "};" << std::endl;
out << std::endl;
}
int get_min_x_advance(const DataFile &datafile)
{
int min = datafile.GetGlyphEntry(0).width;
for (const DataFile::glyphentry_t &g : datafile.GetGlyphTable())
{
if (min > g.width)
min = g.width;
}
return min;
}
int get_max_x_advance(const DataFile &datafile)
{
int max = 0;
for (const DataFile::glyphentry_t &g : datafile.GetGlyphTable())
{
if (max < g.width)
max = g.width;
}
return max;
}
// Select the character to use as a fallback.
int select_fallback_char(const DataFile &datafile)
{
std::set<int> chars;
size_t i = 0;
for (const DataFile::glyphentry_t &g: datafile.GetGlyphTable())
{
for (size_t c: g.chars)
{
chars.insert(c);
}
i++;
}
if (chars.count(0xFFFD))
return 0xFFFD; // Unicode replacement character
if (chars.count(0))
return 0; // Used by many BDF fonts as replacement char
if (chars.count('?'))
return '?';
return ' ';
}
// Decide how to best divide the characters in the font into ranges.
// Limitations are:
// - Gaps longer than minimum_gap should result in separate ranges.
// - Each range can have encoded data size of at most maximum_size.
std::vector<char_range_t> compute_char_ranges(const DataFile &datafile,
std::function<size_t(size_t)> get_encoded_glyph_size,
size_t maximum_size,
size_t minimum_gap)
{
std::vector<char_range_t> result;
std::map<size_t, size_t> char_to_glyph = datafile.GetCharToGlyphMap();
std::vector<size_t> chars;
// Get list of all characters in numeric order.
for (auto iter : char_to_glyph)
chars.push_back(iter.first);
// Pick out ranges until we have processed all characters
size_t i = 0;
while (i < chars.size())
{
char_range_t range;
range.first_char = chars.at(i);
// Find the point where there is a gap larger than minimum_gap.
i++;
while (i < chars.size() && chars.at(i) - chars.at(i - 1) < minimum_gap)
i++;
uint16_t last_char = chars.at(i - 1);
// Then store the indices of glyphs for each character
size_t data_length = 0;
for (size_t j = range.first_char; j <= last_char; j++)
{
if (char_to_glyph.count(j) == 0)
{
// Missing character
range.glyph_indices.push_back(-1);
continue;
}
int glyph_index = char_to_glyph[j];
// Monitor the amount of the data in the range and split it
// if it grows too large.
data_length += get_encoded_glyph_size(glyph_index);
if (data_length > maximum_size)
{
last_char = j - 1;
break;
}
range.glyph_indices.push_back(glyph_index);
}
range.char_count = last_char - range.first_char + 1;
result.push_back(range);
}
return result;
}
}

View File

@ -0,0 +1,52 @@
// Utility functions for exporting to C source code files.
#pragma once
#include <string>
#include <vector>
#include <iostream>
#include <functional>
#include "datafile.hh"
namespace mcufont {
// Convert a file name to a valid C identifier
std::string filename_to_identifier(std::string name);
// Write a vector of integers as line-wrapped hex/integer data for initializing const array.
void wordwrap_vector(std::ostream &out, const std::vector<unsigned> &data,
const std::string &prefix, size_t width = 2);
// Write a vector of integers as a C constant array of given datatype.
void write_const_table(std::ostream &out, const std::vector<unsigned> &data,
const std::string &datatype, const std::string &tablename,
size_t width = 2);
// Get minimum tracking width of font
int get_min_x_advance(const DataFile &datafile);
// Get maximum tracking width of font
int get_max_x_advance(const DataFile &datafile);
// Select the character to use as a fallback.
int select_fallback_char(const DataFile &datafile);
// Structure to represent one consecutive range of characters.
struct char_range_t
{
uint16_t first_char;
uint16_t char_count;
std::vector<int> glyph_indices;
char_range_t(): first_char(0), char_count(0) {}
};
// Decide how to best divide the characters in the font into ranges.
// Limitations are:
// - Gaps longer than minimum_gap should result in separate ranges.
// - Each range can have encoded data size of at most maximum_size.
std::vector<char_range_t> compute_char_ranges(const DataFile &datafile,
std::function<size_t(size_t)> get_encoded_glyph_size,
size_t maximum_size,
size_t minimum_gap);
}

View File

@ -0,0 +1,177 @@
#include "freetype_import.hh"
#include "importtools.hh"
#include <map>
#include <string>
#include <stdexcept>
#include <iostream>
#include "ccfixes.hh"
#include <ft2build.h>
#include FT_FREETYPE_H
#undef __FTERRORS_H__
#define FT_ERRORDEF( e, v, s ) std::make_pair( e, s ),
#define FT_ERROR_START_LIST static const std::map<FT_Error, std::string> ft_errors {
#define FT_ERROR_END_LIST };
#include FT_ERRORS_H
namespace mcufont {
static void checkFT(FT_Error error)
{
if (error != 0)
{
if (ft_errors.count(error))
throw std::runtime_error("libfreetype error " +
std::to_string(error) + ": " + ft_errors.at(error));
else
throw std::runtime_error("unknown libfreetype error " +
std::to_string(error));
}
}
// Automatically allocated & freed wrapper for FT_Library
class _FT_Library
{
public:
_FT_Library() { checkFT(FT_Init_FreeType(&m_lib)); }
~_FT_Library() { checkFT(FT_Done_FreeType(m_lib)); }
operator FT_Library() { return m_lib; }
private:
FT_Library m_lib;
};
// Automatically allocated & freed wrapper for FT_Face
class _FT_Face
{
public:
_FT_Face(FT_Library lib, const std::vector<char> &data)
{
checkFT(FT_New_Memory_Face(lib, (const unsigned char *)&data[0],
data.size(), 0, &m_face));
}
~_FT_Face() { checkFT(FT_Done_Face(m_face)); }
operator FT_Face() { return m_face; }
FT_Face operator->() { return m_face; }
private:
FT_Face m_face;
};
// Read all the data from a file into a memory buffer.
static void readfile(std::istream &file, std::vector<char> &data)
{
while (file.good())
{
const size_t blocksize = 4096;
size_t oldsize = data.size();
data.resize(oldsize + blocksize);
file.read(&data[oldsize], blocksize);
data.resize(oldsize + file.gcount());
}
}
std::unique_ptr<DataFile> LoadFreetype(std::istream &file, int size, bool bw)
{
std::vector<char> data;
readfile(file, data);
_FT_Library lib;
_FT_Face face(lib, data);
checkFT(FT_Set_Pixel_Sizes(face, size, size));
DataFile::fontinfo_t fontinfo = {};
std::vector<DataFile::glyphentry_t> glyphtable;
std::vector<DataFile::dictentry_t> dictionary;
// Convert size to pixels and round to nearest.
int u_per_em = face->units_per_EM;
auto topx = [size, u_per_em](int s) { return (s * size + u_per_em / 2) / u_per_em; };
fontinfo.name = std::string(face->family_name) + " " +
std::string(face->style_name) + " " +
std::to_string(size);
// Reserve 4 pixels on each side for antialiasing + hinting.
// They will be cropped off later.
fontinfo.max_width = topx(face->bbox.xMax - face->bbox.xMin) + 8;
fontinfo.max_height = topx(face->bbox.yMax - face->bbox.yMin) + 8;
fontinfo.baseline_x = topx(-face->bbox.xMin) + 4;
fontinfo.baseline_y = topx(face->bbox.yMax) + 4;
fontinfo.line_height = topx(face->height);
FT_Int32 loadmode = FT_LOAD_TARGET_NORMAL | FT_LOAD_RENDER;
if (bw)
loadmode = FT_LOAD_TARGET_MONO | FT_LOAD_MONOCHROME | FT_LOAD_RENDER;
FT_ULong charcode;
FT_UInt gindex;
charcode = FT_Get_First_Char(face, &gindex);
while (gindex)
{
try
{
checkFT(FT_Load_Glyph(face, gindex, loadmode));
}
catch (std::runtime_error &e)
{
std::cerr << "Skipping glyph " << gindex << ": " << e.what() << std::endl;
charcode = FT_Get_Next_Char(face, charcode, &gindex);
}
DataFile::glyphentry_t glyph;
glyph.width = (face->glyph->advance.x + 32) / 64;
glyph.chars.push_back(charcode);
glyph.data.resize(fontinfo.max_width * fontinfo.max_height);
int w = face->glyph->bitmap.width;
int dw = fontinfo.max_width;
int dx = fontinfo.baseline_x + face->glyph->bitmap_left;
int dy = fontinfo.baseline_y - face->glyph->bitmap_top;
/* Some combining diacritics seem to exceed the bounding box.
* We don't support them all that well anyway, so just move
* them inside the box in order not to crash.. */
if (dy < 0)
dy = 0;
if (dy + face->glyph->bitmap.rows > fontinfo.max_height)
dy = fontinfo.max_height - face->glyph->bitmap.rows;
size_t s = face->glyph->bitmap.pitch;
for (int y = 0; y < face->glyph->bitmap.rows; y++)
{
for (int x = 0; x < face->glyph->bitmap.width; x++)
{
size_t index = (y + dy) * dw + x + dx;
if (face->glyph->bitmap.pixel_mode == FT_PIXEL_MODE_MONO)
{
uint8_t byte = face->glyph->bitmap.buffer[s * y + x / 8];
byte <<= x % 8;
glyph.data.at(index) = (byte & 0x80) ? 15 : 0;
}
else
{
glyph.data.at(index) =
(face->glyph->bitmap.buffer[w * y + x] + 8) / 17;
}
}
}
glyphtable.push_back(glyph);
charcode = FT_Get_Next_Char(face, charcode, &gindex);
}
eliminate_duplicates(glyphtable);
crop_glyphs(glyphtable, fontinfo);
detect_flags(glyphtable, fontinfo);
std::unique_ptr<DataFile> result(new DataFile(
dictionary, glyphtable, fontinfo));
return result;
}
}

View File

@ -0,0 +1,10 @@
// Function for importing any font supported by libfreetype.
#pragma once
#include "datafile.hh"
namespace mcufont {
std::unique_ptr<DataFile> LoadFreetype(std::istream &file, int size, bool bw);
}

View File

@ -0,0 +1,134 @@
#include "importtools.hh"
#include <limits>
namespace mcufont {
void eliminate_duplicates(std::vector<DataFile::glyphentry_t> &glyphtable)
{
for (size_t i = 0; i + 1 < glyphtable.size(); i++)
{
for (size_t j = i + 1; j < glyphtable.size(); j++)
{
if (glyphtable.at(i).data == glyphtable.at(j).data &&
glyphtable.at(i).width == glyphtable.at(j).width)
{
for (int c : glyphtable.at(j).chars)
glyphtable.at(i).chars.push_back(c);
glyphtable.erase(glyphtable.begin() + j);
j--;
}
}
}
}
struct bbox_t
{
int left;
int top;
int right;
int bottom;
bbox_t()
{
left = std::numeric_limits<int>::max();
top = std::numeric_limits<int>::max();
right = std::numeric_limits<int>::min();
bottom = std::numeric_limits<int>::min();
}
void update(int x, int y)
{
if (x < left) left = x;
if (x > right) right = x;
if (y < top) top = y;
if (y > bottom) bottom = y;
}
};
void crop_glyphs(std::vector<DataFile::glyphentry_t> &glyphtable,
DataFile::fontinfo_t &fontinfo)
{
// Find out the maximum bounding box
bbox_t bbox;
for (DataFile::glyphentry_t &glyph : glyphtable)
{
for (int y = 0; y < fontinfo.max_height; y++)
{
for (int x = 0; x < fontinfo.max_width; x++)
{
if (glyph.data.at(y * fontinfo.max_width + x))
bbox.update(x, y);
}
}
}
// Crop the glyphs to that
size_t old_w = fontinfo.max_width;
size_t new_w = bbox.right - bbox.left + 1;
size_t new_h = bbox.bottom - bbox.top + 1;
for (DataFile::glyphentry_t &glyph : glyphtable)
{
DataFile::pixels_t old = glyph.data;
glyph.data.clear();
for (size_t y = 0; y < new_h; y++)
{
for (size_t x = 0; x < new_w; x++)
{
size_t old_x = bbox.left + x;
size_t old_y = bbox.top + y;
size_t old_pos = old_w * old_y + old_x;
glyph.data.push_back(old.at(old_pos));
}
}
}
fontinfo.max_width = new_w;
fontinfo.max_height = new_h;
fontinfo.baseline_x -= bbox.left;
fontinfo.baseline_y -= bbox.top;
}
void detect_flags(const std::vector<DataFile::glyphentry_t> &glyphtable,
DataFile::fontinfo_t &fontinfo)
{
if (!glyphtable.size())
return;
// Check if all glyphs have equal width
int width = glyphtable[0].width;
bool is_monospace = true;
for (const DataFile::glyphentry_t &g : glyphtable)
{
if (g.width != width)
{
is_monospace = false;
break;
}
}
if (is_monospace)
fontinfo.flags |= DataFile::FLAG_MONOSPACE;
// Check if all glyphs contain only 0 or 15 alpha
bool is_bw = true;
for (const DataFile::glyphentry_t &g : glyphtable)
{
for (uint8_t pixel : g.data)
{
if (pixel != 0 && pixel != 15)
{
is_bw = false;
break;
}
}
if (!is_bw) break;
}
if (is_bw)
fontinfo.flags |= DataFile::FLAG_BW;
}
}

View File

@ -0,0 +1,20 @@
// Utility functions for processing imported font files.
#pragma once
#include "datafile.hh"
namespace mcufont {
// Find and eliminate any duplicate glyphs by appending their char vectors.
void eliminate_duplicates(std::vector<DataFile::glyphentry_t> &glyphtable);
// Calculate the maximum bounding box of the glyphs and crop them to that.
// Adjust fontinfo accordingly.
void crop_glyphs(std::vector<DataFile::glyphentry_t> &glyphtable,
DataFile::fontinfo_t &fontinfo);
// Fill in the flags (BW, monospace) automatically.
void detect_flags(const std::vector<DataFile::glyphentry_t> &glyphtable,
DataFile::fontinfo_t &fontinfo);
}

Binary file not shown.

View File

@ -0,0 +1,480 @@
#include "datafile.hh"
#include "importtools.hh"
#include "bdf_import.hh"
#include "freetype_import.hh"
#include "export_rlefont.hh"
#include "encode_rlefont.hh"
#include "optimize_rlefont.hh"
#include "export_bwfont.hh"
#include <vector>
#include <string>
#include <set>
#include <fstream>
#include <iostream>
#include <iomanip>
#include <cstdlib>
#include <ctime>
#include <map>
#include "ccfixes.hh"
using namespace mcufont;
static std::string strip_extension(std::string filename)
{
size_t pos = filename.find_last_of('.');
if (pos == std::string::npos)
{
return filename;
}
else
{
return filename.substr(0, pos);
}
}
static std::unique_ptr<DataFile> load_dat(std::string src)
{
std::ifstream infile(src);
if (!infile.good())
{
std::cerr << "Could not open " << src << std::endl;
return nullptr;
}
std::unique_ptr<DataFile> f = DataFile::Load(infile);
if (!f)
{
std::cerr << "Invalid format for .dat file: " << src << std::endl;
return nullptr;
}
return f;
}
static bool save_dat(std::string dest, DataFile *f)
{
std::ofstream outfile(dest);
if (!outfile.good())
{
std::cerr << "Could not open " << dest << std::endl;
return false;
}
f->Save(outfile);
if (!outfile.good())
{
std::cerr << "Could not write to " << dest << std::endl;
return false;
}
return true;
}
enum status_t
{
STATUS_OK = 0, // All good
STATUS_INVALID = 1, // Invalid command or args
STATUS_ERROR = 2 // Error when executing command
};
static status_t cmd_import_ttf(const std::vector<std::string> &args)
{
if (args.size() != 3 && args.size() != 4)
return STATUS_INVALID;
std::string src = args.at(1);
int size = std::stoi(args.at(2));
bool bw = (args.size() == 4 && args.at(3) == "bw");
std::string dest = strip_extension(src) + std::to_string(size) + (bw ? "bw" : "") + ".dat";
std::ifstream infile(src);
if (!infile.good())
{
std::cerr << "Could not open " << src << std::endl;
return STATUS_ERROR;
}
std::cout << "Importing " << src << " to " << dest << std::endl;
std::unique_ptr<DataFile> f = LoadFreetype(infile, size, bw);
mcufont::rlefont::init_dictionary(*f);
if (!save_dat(dest, f.get()))
return STATUS_ERROR;
std::cout << "Done: " << f->GetGlyphCount() << " unique glyphs." << std::endl;
return STATUS_OK;
}
static status_t cmd_import_bdf(const std::vector<std::string> &args)
{
if (args.size() != 2)
return STATUS_INVALID;
std::string src = args.at(1);
std::string dest = strip_extension(args.at(1)) + ".dat";
std::ifstream infile(src);
if (!infile.good())
{
std::cerr << "Could not open " << src << std::endl;
return STATUS_ERROR;
}
std::cout << "Importing " << src << " to " << dest << std::endl;
std::unique_ptr<DataFile> f = LoadBDF(infile);
mcufont::rlefont::init_dictionary(*f);
if (!save_dat(dest, f.get()))
return STATUS_ERROR;
std::cout << "Done: " << f->GetGlyphCount() << " unique glyphs." << std::endl;
return STATUS_OK;
}
static status_t cmd_filter(const std::vector<std::string> &args)
{
if (args.size() < 3)
return STATUS_INVALID;
std::set<int> allowed;
// Parse arguments
for (size_t i = 2; i < args.size(); i++)
{
std::string s = args.at(i);
size_t pos = s.find('-');
if (pos == std::string::npos)
{
// Single char
allowed.insert(std::stoi(s, nullptr, 0));
}
else
{
// Range
int start = std::stoi(s.substr(0, pos), nullptr, 0);
int end = std::stoi(s.substr(pos + 1), nullptr, 0);
for (int j = start; j <= end; j++)
{
allowed.insert(j);
}
}
}
std::string src = args.at(1);
std::unique_ptr<DataFile> f = load_dat(src);
if (!f)
return STATUS_ERROR;
std::cout << "Font originally had " << f->GetGlyphCount() << " glyphs." << std::endl;
// Filter the glyphs
std::vector<DataFile::glyphentry_t> newglyphs;
for (size_t i = 0; i < f->GetGlyphCount(); i++)
{
DataFile::glyphentry_t g = f->GetGlyphEntry(i);
for (size_t j = 0; j < g.chars.size(); j++)
{
if (!allowed.count(g.chars.at(j)))
{
g.chars.erase(g.chars.begin() + j);
j--;
}
}
if (g.chars.size())
{
newglyphs.push_back(g);
}
}
DataFile::fontinfo_t fontinfo = f->GetFontInfo();
crop_glyphs(newglyphs, fontinfo);
detect_flags(newglyphs, fontinfo);
f.reset(new DataFile(f->GetDictionary(), newglyphs, fontinfo));
std::cout << "After filtering, " << f->GetGlyphCount() << " glyphs remain." << std::endl;
if (!save_dat(src, f.get()))
return STATUS_ERROR;
return STATUS_OK;
}
static status_t cmd_show_glyph(const std::vector<std::string> &args)
{
if (args.size() != 3)
return STATUS_INVALID;
std::string src = args.at(1);
std::unique_ptr<DataFile> f = load_dat(src);
if (!f)
return STATUS_ERROR;
size_t index = 0;
if (args.at(2) == "largest")
{
std::unique_ptr<mcufont::rlefont::encoded_font_t> e =
mcufont::rlefont::encode_font(*f, false);
size_t maxlen = 0;
size_t i = 0;
for (mcufont::rlefont::encoded_font_t::refstring_t g : e->glyphs)
{
if (g.size() > maxlen)
{
maxlen = g.size();
index = i;
}
i++;
}
std::cout << "Index " << index << ", length " << maxlen << std::endl;
}
else
{
index = strtol(args.at(2).c_str(), nullptr, 0);
}
if (index < 0 || index >= f->GetGlyphCount())
{
std::cerr << "No such glyph " << index << std::endl;
return STATUS_ERROR;
}
std::cout << "Width: " << f->GetGlyphEntry(index).width << std::endl;
std::cout << "Chars: ";
for (int c: f->GetGlyphEntry(index).chars) std::cout << c << " ";
std::cout << std::endl;
std::cout << f->GlyphToText(index);
return STATUS_OK;
}
static status_t cmd_rlefont_export(const std::vector<std::string> &args)
{
if (args.size() != 2 && args.size() != 3)
return STATUS_INVALID;
std::string src = args.at(1);
std::string dst = (args.size() == 2) ? strip_extension(src) + ".c" : args.at(2);
std::unique_ptr<DataFile> f = load_dat(src);
if (!f)
return STATUS_ERROR;
{
std::ofstream source(dst);
mcufont::rlefont::write_source(source, dst, *f);
std::cout << "Wrote " << dst << std::endl;
}
return STATUS_OK;
}
static status_t cmd_rlefont_size(const std::vector<std::string> &args)
{
if (args.size() != 2)
return STATUS_INVALID;
std::string src = args.at(1);
std::unique_ptr<DataFile> f = load_dat(src);
if (!f)
return STATUS_ERROR;
size_t size = mcufont::rlefont::get_encoded_size(*f);
std::cout << "Glyph count: " << f->GetGlyphCount() << std::endl;
std::cout << "Glyph bbox: " << f->GetFontInfo().max_width << "x"
<< f->GetFontInfo().max_height << " pixels" << std::endl;
std::cout << "Uncompressed size: " << f->GetGlyphCount() *
f->GetFontInfo().max_width * f->GetFontInfo().max_height / 2
<< " bytes" << std::endl;
std::cout << "Compressed size: " << size << " bytes" << std::endl;
std::cout << "Bytes per glyph: " << size / f->GetGlyphCount() << std::endl;
return STATUS_OK;
}
static status_t cmd_rlefont_optimize(const std::vector<std::string> &args)
{
if (args.size() != 2 && args.size() != 3)
return STATUS_INVALID;
std::string src = args.at(1);
std::unique_ptr<DataFile> f = load_dat(src);
if (!f)
return STATUS_ERROR;
size_t oldsize = mcufont::rlefont::get_encoded_size(*f);
std::cout << "Original size is " << oldsize << " bytes" << std::endl;
std::cout << "Press ctrl-C at any time to stop." << std::endl;
std::cout << "Results are saved automatically after each iteration." << std::endl;
int limit = 100;
if (args.size() == 3)
{
limit = std::stoi(args.at(2));
}
if (limit > 0)
std::cout << "Limit is " << limit << " iterations" << std::endl;
int i = 0;
time_t oldtime = time(NULL);
while (!limit || i < limit)
{
mcufont::rlefont::optimize(*f);
size_t newsize = mcufont::rlefont::get_encoded_size(*f);
time_t newtime = time(NULL);
int bytes_per_min = (oldsize - newsize) * 60 / (newtime - oldtime + 1);
i++;
std::cout << "iteration " << i << ", size " << newsize
<< " bytes, speed " << bytes_per_min << " B/min"
<< std::endl;
{
if (!save_dat(src, f.get()))
return STATUS_ERROR;
}
}
return STATUS_OK;
}
static status_t cmd_rlefont_show_encoded(const std::vector<std::string> &args)
{
if (args.size() != 2)
return STATUS_INVALID;
std::string src = args.at(1);
std::unique_ptr<DataFile> f = load_dat(src);
if (!f)
return STATUS_ERROR;
std::unique_ptr<mcufont::rlefont::encoded_font_t> e =
mcufont::rlefont::encode_font(*f, false);
int i = 0;
for (mcufont::rlefont::encoded_font_t::rlestring_t d : e->rle_dictionary)
{
std::cout << "Dict RLE " << 24 + i++ << ": ";
for (uint8_t v : d)
std::cout << std::setfill('0') << std::setw(2) << std::hex << (int)v << " ";
std::cout << std::endl;
}
for (mcufont::rlefont::encoded_font_t::refstring_t d : e->ref_dictionary)
{
std::cout << "Dict Ref " << 24 + i++ << ": ";
for (uint8_t v : d)
std::cout << std::setfill('0') << std::setw(2) << std::hex << (int)v << " ";
std::cout << std::endl;
}
i = 0;
for (mcufont::rlefont::encoded_font_t::refstring_t g : e->glyphs)
{
std::cout << "Glyph " << i++ << ": ";
for (uint8_t v : g)
std::cout << std::setfill('0') << std::setw(2) << std::hex << (int)v << " ";
std::cout << std::endl;
}
return STATUS_OK;
}
static status_t cmd_bwfont_export(const std::vector<std::string> &args)
{
if (args.size() != 2 && args.size() != 3)
return STATUS_INVALID;
std::string src = args.at(1);
std::string dst = (args.size() == 2) ? strip_extension(src) + ".c" : args.at(2);
std::unique_ptr<DataFile> f = load_dat(src);
if (!f)
return STATUS_ERROR;
if (!(f->GetFontInfo().flags & DataFile::FLAG_BW))
{
std::cout << "Warning: font is not black and white" << std::endl;
}
{
std::ofstream source(dst);
mcufont::bwfont::write_source(source, dst, *f);
std::cout << "Wrote " << dst << std::endl;
}
return STATUS_OK;
}
static const char *usage_msg =
"Usage: mcufont <command> [options] ...\n"
"Commands for importing:\n"
" import_ttf <ttffile> <size> [bw] Import a .ttf font into a data file.\n"
" import_bdf <bdffile> Import a .bdf font into a data file.\n"
"\n"
"Commands for inspecting and editing data files:\n"
" filter <datfile> <range> ... Remove everything except specified characters.\n"
" show_glyph <datfile> <index> Show the glyph at index.\n"
"\n"
"Commands specific to rlefont format:\n"
" rlefont_size <datfile> Check the encoded size of the data file.\n"
" rlefont_optimize <datfile> Perform an optimization pass on the data file.\n"
" rlefont_export <datfile> [outfile] Export to .c source code.\n"
" rlefont_show_encoded <datfile> Show the encoded data for debugging.\n"
"\n"
"Commands specific to bwfont format:\n"
" bwfont_export <datfile> [outfile Export to .c source code.\n"
"";
typedef status_t (*cmd_t)(const std::vector<std::string> &args);
static const std::map<std::string, cmd_t> command_list {
{"import_ttf", cmd_import_ttf},
{"import_bdf", cmd_import_bdf},
{"filter", cmd_filter},
{"show_glyph", cmd_show_glyph},
{"rlefont_size", cmd_rlefont_size},
{"rlefont_optimize", cmd_rlefont_optimize},
{"rlefont_export", cmd_rlefont_export},
{"rlefont_show_encoded", cmd_rlefont_show_encoded},
{"bwfont_export", cmd_bwfont_export},
};
int main(int argc, char **argv)
{
std::vector<std::string> args;
for (int i = 1; i < argc; i++)
args.push_back(argv[i]);
status_t status = STATUS_INVALID;
if (args.size() >= 1 && command_list.count(args.at(0)))
{
status = command_list.find(args.at(0))->second(args);
}
if (status == STATUS_INVALID)
{
std::cout << usage_msg << std::endl;
}
return status;
}

View File

@ -0,0 +1,417 @@
#include "optimize_rlefont.hh"
#include "encode_rlefont.hh"
#include <random>
#include <iostream>
#include <set>
#include <thread>
#include <algorithm>
#include "ccfixes.hh"
namespace mcufont {
namespace rlefont {
typedef std::mt19937 rnd_t;
// Select a random substring among all the glyphs in the datafile.
std::unique_ptr<DataFile::pixels_t> random_substring(const DataFile &datafile, rnd_t &rnd)
{
std::uniform_int_distribution<size_t> dist1(0, datafile.GetGlyphCount() - 1);
size_t index = dist1(rnd);
const DataFile::pixels_t &pixels = datafile.GetGlyphEntry(index).data;
std::uniform_int_distribution<size_t> dist2(2, pixels.size());
size_t length = dist2(rnd);
std::uniform_int_distribution<size_t> dist3(0, pixels.size() - length);
size_t start = dist3(rnd);
std::unique_ptr<DataFile::pixels_t> result;
result.reset(new DataFile::pixels_t(pixels.begin() + start,
pixels.begin() + start + length));
return result;
}
// Try to replace the worst dictionary entry with a better one.
void optimize_worst(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose)
{
std::uniform_int_distribution<size_t> dist(0, 1);
DataFile trial = datafile;
size_t worst = trial.GetLowScoreIndex();
DataFile::dictentry_t d = trial.GetDictionaryEntry(worst);
d.replacement = *random_substring(datafile, rnd);
d.ref_encode = dist(rnd);
trial.SetDictionaryEntry(worst, d);
size_t newsize = get_encoded_size(trial);
if (newsize < size)
{
d.score = size - newsize;
datafile.SetDictionaryEntry(worst, d);
size = newsize;
if (verbose)
std::cout << "optimize_worst: replaced " << worst
<< " score " << d.score << std::endl;
}
}
// Try to replace random dictionary entry with another one.
void optimize_any(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose)
{
DataFile trial = datafile;
std::uniform_int_distribution<size_t> dist(0, DataFile::dictionarysize - 1);
size_t index = dist(rnd);
DataFile::dictentry_t d = trial.GetDictionaryEntry(index);
d.replacement = *random_substring(datafile, rnd);
trial.SetDictionaryEntry(index, d);
size_t newsize = get_encoded_size(trial);
if (newsize < size)
{
d.score = size - newsize;
datafile.SetDictionaryEntry(index, d);
size = newsize;
if (verbose)
std::cout << "optimize_any: replaced " << index
<< " score " << d.score << std::endl;
}
}
// Try to append or prepend random dictionary entry.
void optimize_expand(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose, bool binary_only)
{
DataFile trial = datafile;
std::uniform_int_distribution<size_t> dist1(0, DataFile::dictionarysize - 1);
size_t index = dist1(rnd);
DataFile::dictentry_t d = trial.GetDictionaryEntry(index);
std::uniform_int_distribution<size_t> dist3(1, 3);
size_t count = dist3(rnd);
for (size_t i = 0; i < count; i++)
{
std::uniform_int_distribution<size_t> booldist(0, 1);
std::uniform_int_distribution<size_t> pixeldist(0, 15);
uint8_t pixel;
if (binary_only)
{
pixel = booldist(rnd) ? 15 : 0;
}
else
{
pixel = pixeldist(rnd);
}
bool prepend = booldist(rnd);
if (prepend)
{
d.replacement.insert(d.replacement.begin(), pixel);
}
else
{
d.replacement.push_back(pixel);
}
}
trial.SetDictionaryEntry(index, d);
size_t newsize = get_encoded_size(trial);
if (newsize < size)
{
d.score = size - newsize;
datafile.SetDictionaryEntry(index, d);
size = newsize;
if (verbose)
std::cout << "optimize_expand: expanded " << index
<< " by " << count << " pixels, score " << d.score << std::endl;
}
}
// Try to trim random dictionary entry.
void optimize_trim(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose)
{
DataFile trial = datafile;
std::uniform_int_distribution<size_t> dist1(0, DataFile::dictionarysize - 1);
size_t index = dist1(rnd);
DataFile::dictentry_t d = trial.GetDictionaryEntry(index);
if (d.replacement.size() <= 2) return;
std::uniform_int_distribution<size_t> dist2(0, std::min((int)d.replacement.size() / 2, 5));
size_t start = dist2(rnd);
size_t end = dist2(rnd);
if (start)
{
d.replacement.erase(d.replacement.begin(), d.replacement.begin() + start);
}
if (end)
{
d.replacement.erase(d.replacement.end() - end, d.replacement.end() - 1);
}
trial.SetDictionaryEntry(index, d);
size_t newsize = get_encoded_size(trial);
if (newsize < size)
{
d.score = size - newsize;
datafile.SetDictionaryEntry(index, d);
size = newsize;
if (verbose)
std::cout << "optimize_trim: trimmed " << index
<< " by " << start << " pixels from start and "
<< end << " pixels from end, score " << d.score << std::endl;
}
}
// Switch random dictionary entry to use ref encoding or back to rle.
void optimize_refdict(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose)
{
DataFile trial = datafile;
std::uniform_int_distribution<size_t> dist1(0, DataFile::dictionarysize - 1);
size_t index = dist1(rnd);
DataFile::dictentry_t d = trial.GetDictionaryEntry(index);
d.ref_encode = !d.ref_encode;
trial.SetDictionaryEntry(index, d);
size_t newsize = get_encoded_size(trial);
if (newsize < size)
{
d.score = size - newsize;
datafile.SetDictionaryEntry(index, d);
size = newsize;
if (verbose)
std::cout << "optimize_refdict: switched " << index
<< " to " << (d.ref_encode ? "ref" : "RLE")
<< ", score " << d.score << std::endl;
}
}
// Combine two random dictionary entries.
void optimize_combine(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose)
{
DataFile trial = datafile;
std::uniform_int_distribution<size_t> dist1(0, DataFile::dictionarysize - 1);
size_t worst = datafile.GetLowScoreIndex();
size_t index1 = dist1(rnd);
size_t index2 = dist1(rnd);
const DataFile::pixels_t &part1 = datafile.GetDictionaryEntry(index1).replacement;
const DataFile::pixels_t &part2 = datafile.GetDictionaryEntry(index2).replacement;
DataFile::dictentry_t d;
d.replacement = part1;
d.replacement.insert(d.replacement.end(), part2.begin(), part2.end());
d.ref_encode = true;
trial.SetDictionaryEntry(worst, d);
size_t newsize = get_encoded_size(trial);
if (newsize < size)
{
d.score = size - newsize;
datafile.SetDictionaryEntry(worst, d);
size = newsize;
if (verbose)
std::cout << "optimize_combine: combined " << index1
<< " and " << index2 << " to replace " << worst
<< ", score " << d.score << std::endl;
}
}
// Pick a random part of an encoded glyph and encode it as a ref dict.
void optimize_encpart(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose)
{
std::unique_ptr<encoded_font_t> e = encode_font(datafile);
// Pick a random encoded glyph
std::uniform_int_distribution<size_t> dist1(0, datafile.GetGlyphCount() - 1);
size_t index = dist1(rnd);
const encoded_font_t::refstring_t &refstr = e->glyphs.at(index);
if (refstr.size() < 2)
return;
// Pick a random part of it
std::uniform_int_distribution<size_t> dist2(2, refstr.size());
size_t length = dist2(rnd);
std::uniform_int_distribution<size_t> dist3(0, refstr.size() - length);
size_t start = dist3(rnd);
// Decode that part
encoded_font_t::refstring_t substr(refstr.begin() + start,
refstr.begin() + start + length);
std::unique_ptr<DataFile::pixels_t> decoded =
decode_glyph(*e, substr, datafile.GetFontInfo());
// Add that as a new dictionary entry
DataFile trial = datafile;
size_t worst = trial.GetLowScoreIndex();
DataFile::dictentry_t d = trial.GetDictionaryEntry(worst);
d.replacement = *decoded;
d.ref_encode = true;
trial.SetDictionaryEntry(worst, d);
size_t newsize = get_encoded_size(trial);
if (newsize < size)
{
d.score = size - newsize;
datafile.SetDictionaryEntry(worst, d);
size = newsize;
if (verbose)
std::cout << "optimize_encpart: replaced " << worst
<< " score " << d.score << std::endl;
}
}
// Execute all the optimization algorithms once.
void optimize_pass(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose)
{
optimize_worst(datafile, size, rnd, verbose);
optimize_any(datafile, size, rnd, verbose);
optimize_expand(datafile, size, rnd, verbose, false);
optimize_expand(datafile, size, rnd, verbose, true);
optimize_trim(datafile, size, rnd, verbose);
optimize_refdict(datafile, size, rnd, verbose);
optimize_combine(datafile, size, rnd, verbose);
optimize_encpart(datafile, size, rnd, verbose);
}
// Execute multiple passes in parallel and take the one with the best result.
// The amount of parallelism is hardcoded in order to retain deterministic
// behaviour.
void optimize_parallel(DataFile &datafile, size_t &size, rnd_t &rnd, bool verbose, int num_threads = 4)
{
std::vector<DataFile> datafiles;
std::vector<size_t> sizes;
std::vector<rnd_t> rnds;
std::vector<std::unique_ptr<std::thread> > threads;
for (int i = 0; i < num_threads; i++)
{
datafiles.emplace_back(datafile);
sizes.emplace_back(size);
rnds.emplace_back(rnd());
}
for (int i = 0; i < num_threads; i++)
{
threads.emplace_back(new std::thread(optimize_pass,
std::ref(datafiles.at(i)),
std::ref(sizes.at(i)),
std::ref(rnds.at(i)),
verbose));
}
for (int i = 0; i < num_threads; i++)
{
threads.at(i)->join();
}
int best = std::min_element(sizes.begin(), sizes.end()) - sizes.begin();
size = sizes.at(best);
datafile = datafiles.at(best);
}
// Go through all the dictionary entries and check what it costs to remove
// them. Removes any entries with negative or zero score.
void update_scores(DataFile &datafile, bool verbose)
{
size_t oldsize = get_encoded_size(datafile);
for (size_t i = 0; i < DataFile::dictionarysize; i++)
{
DataFile trial = datafile;
DataFile::dictentry_t dummy = {};
trial.SetDictionaryEntry(i, dummy);
size_t newsize = get_encoded_size(trial);
DataFile::dictentry_t d = datafile.GetDictionaryEntry(i);
d.score = newsize - oldsize;
if (d.score > 0)
{
datafile.SetDictionaryEntry(i, d);
}
else
{
datafile.SetDictionaryEntry(i, dummy);
if (verbose && d.replacement.size() != 0)
std::cout << "update_scores: dropped " << i
<< " score " << -d.score << std::endl;
}
}
}
void init_dictionary(DataFile &datafile)
{
rnd_t rnd(datafile.GetSeed());
if (datafile.GetGlyphCount() == 0)
return;
std::set<DataFile::pixels_t> seen_substrings;
std::set<DataFile::pixels_t> added_substrings;
size_t i = 0;
while (i < DataFile::dictionarysize)
{
DataFile::pixels_t substring = *random_substring(datafile, rnd);
if (!seen_substrings.count(substring))
{
seen_substrings.insert(substring);
}
else if (!added_substrings.count(substring))
{
// When we see a substring second time, add it.
DataFile::dictentry_t d;
d.score = 0;
d.replacement = substring;
datafile.SetDictionaryEntry(i, d);
i++;
added_substrings.insert(substring);
}
}
}
void optimize(DataFile &datafile, size_t iterations)
{
bool verbose = false;
rnd_t rnd(datafile.GetSeed());
update_scores(datafile, verbose);
size_t size = get_encoded_size(datafile);
for (size_t i = 0; i < iterations; i++)
{
optimize_parallel(datafile, size, rnd, verbose);
}
std::uniform_int_distribution<size_t> dist(0, std::numeric_limits<uint32_t>::max());
datafile.SetSeed(dist(rnd));
}
}}

View File

@ -0,0 +1,15 @@
// This implements the actual optimization passes of the compressor.
#include "datafile.hh"
namespace mcufont {
namespace rlefont {
// Initialize the dictionary table with reasonable guesses.
void init_dictionary(DataFile &datafile);
// Perform a single optimization step, consisting itself of multiple passes
// of each of the optimization algorithms.
void optimize(DataFile &datafile, size_t iterations = 50);
}}