Merging with master for having minz png compression.

Much greater performance boost from better compression algorithm than that from the merged polygons.
This commit is contained in:
tamasmeszaros 2019-03-29 15:21:46 +01:00
commit 725f115756
344 changed files with 786 additions and 116524 deletions

View file

@ -161,6 +161,8 @@ add_library(libslic3r STATIC
utils.cpp
Utils.hpp
MTUtils.hpp
Zipper.hpp
Zipper.cpp
SLA/SLABoilerPlate.hpp
SLA/SLABasePool.hpp
SLA/SLABasePool.cpp
@ -177,8 +179,8 @@ if (SLIC3R_PCH AND NOT SLIC3R_SYNTAXONLY)
add_precompiled_header(libslic3r pchheader.hpp FORCEINCLUDE)
endif ()
target_compile_definitions(libslic3r PUBLIC -DUSE_TBB ${PNG_DEFINITIONS})
target_include_directories(libslic3r PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${LIBNEST2D_INCLUDES} ${PNG_INCLUDE_DIRS} PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
target_compile_definitions(libslic3r PUBLIC -DUSE_TBB)
target_include_directories(libslic3r PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${LIBNEST2D_INCLUDES} PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
target_link_libraries(libslic3r
libnest2d
admesh
@ -188,7 +190,6 @@ target_link_libraries(libslic3r
nowide
${EXPAT_LIBRARIES}
${GLEW_LIBRARIES}
${PNG_LIBRARIES}
glu-libtess
polypartition
poly2tri

View file

@ -4,6 +4,7 @@
#include <boost/log/trivial.hpp>
#include <boost/format.hpp>
#include <boost/filesystem.hpp>
#include <boost/nowide/convert.hpp>
#ifdef WIN32
@ -11,6 +12,7 @@
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <Windows.h>
#include <shellapi.h>
// https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/
// This routine appends the given argument to a command line such that CommandLineToArgvW will return the argument string unchanged.

View file

@ -81,9 +81,8 @@ public:
// Provokes static_assert in the right way.
template<class T = void> struct VeryFalse { static const bool value = false; };
// This has to be explicitly implemented in the gui layer or a default zlib
// based implementation is needed. I don't have time for that and I'm delegating
// the implementation to the gui layer where the gui toolkit can cover this.
// This can be explicitly implemented in the gui layer or the default Zipper
// API in libslic3r with minz.
template<class Fmt> class LayerWriter {
public:
@ -92,17 +91,28 @@ public:
"No layer writer implementation provided!");
}
// Should create a new file within the zip with the given filename. It
// should also finish any previous entry.
void next_entry(const std::string& /*fname*/) {}
// Should create a new file within the archive and write the provided data.
void binary_entry(const std::string& /*fname*/,
const std::uint8_t* buf, size_t len);
// Get the name of the archive but only the name part without the path or
// the extension.
std::string get_name() { return ""; }
// Test whether the object can still be used for writing.
bool is_ok() { return false; }
template<class T> LayerWriter& operator<<(const T& /*arg*/) {
// Write some data (text) into the current file (entry) within the archive.
template<class T> LayerWriter& operator<<(T&& /*arg*/) {
return *this;
}
void close() {}
// Flush the current entry into the archive.
void finalize() {}
};
// Implementation for PNG raster output
@ -111,14 +121,14 @@ public:
template<> class FilePrinter<FilePrinterFormat::SLA_PNGZIP>
{
struct Layer {
Raster first;
std::stringstream second;
Raster raster;
RawBytes rawbytes;
Layer() {}
Layer(const Layer&) = delete;
Layer(Layer&& m):
first(std::move(m.first))/*, second(std::move(m.second))*/ {}
raster(std::move(m.raster)) {}
};
// We will save the compressed PNG data into stringstreams which can be done
@ -136,14 +146,11 @@ template<> class FilePrinter<FilePrinterFormat::SLA_PNGZIP>
int m_cnt_fast_layers = 0;
std::string createIniContent(const std::string& projectname) {
// double layer_height = m_layer_height;
using std::string;
using std::to_string;
auto expt_str = to_string(m_exp_time_s);
auto expt_first_str = to_string(m_exp_time_first_s);
// auto stepnum_str = to_string(static_cast<unsigned>(800*layer_height));
auto layerh_str = to_string(m_layer_height);
const std::string cnt_fade_layers = to_string(m_cnt_fade_layers);
@ -212,7 +219,7 @@ public:
inline void draw_polygon(const ExPolygon& p, unsigned lyr) {
assert(lyr < m_layers_rst.size());
m_layers_rst[lyr].first.draw(p);
m_layers_rst[lyr].raster.draw(p);
}
inline void draw_polygon(const ClipperLib::Polygon& p, unsigned lyr) {
@ -222,26 +229,26 @@ public:
inline void begin_layer(unsigned lyr) {
if(m_layers_rst.size() <= lyr) m_layers_rst.resize(lyr+1);
m_layers_rst[lyr].first.reset(m_res, m_pxdim, m_o);
m_layers_rst[lyr].raster.reset(m_res, m_pxdim, m_o);
}
inline void begin_layer() {
m_layers_rst.emplace_back();
m_layers_rst.front().first.reset(m_res, m_pxdim, m_o);
m_layers_rst.front().raster.reset(m_res, m_pxdim, m_o);
}
inline void finish_layer(unsigned lyr_id) {
assert(lyr_id < m_layers_rst.size());
m_layers_rst[lyr_id].first.save(m_layers_rst[lyr_id].second,
Raster::Compression::PNG);
m_layers_rst[lyr_id].first.reset();
m_layers_rst[lyr_id].rawbytes =
m_layers_rst[lyr_id].raster.save(Raster::Compression::PNG);
m_layers_rst[lyr_id].raster.reset();
}
inline void finish_layer() {
if(!m_layers_rst.empty()) {
m_layers_rst.back().first.save(m_layers_rst.back().second,
Raster::Compression::PNG);
m_layers_rst.back().first.reset();
m_layers_rst.back().rawbytes =
m_layers_rst.back().raster.save(Raster::Compression::PNG);
m_layers_rst.back().raster.reset();
}
}
@ -260,20 +267,19 @@ public:
for(unsigned i = 0; i < m_layers_rst.size() && writer.is_ok(); i++)
{
if(m_layers_rst[i].second.rdbuf()->in_avail() > 0) {
if(m_layers_rst[i].rawbytes.size() > 0) {
char lyrnum[6];
std::sprintf(lyrnum, "%.5d", i);
auto zfilename = project + lyrnum + ".png";
writer.next_entry(zfilename);
if(!writer.is_ok()) break;
writer << m_layers_rst[i].second.str();
// writer << m_layers_rst[i].second.rdbuf();
// we can keep the date for later calls of this method
//m_layers_rst[i].second.str("");
writer.binary_entry(zfilename,
m_layers_rst[i].rawbytes.data(),
m_layers_rst[i].rawbytes.size());
}
}
writer.finalize();
} catch(std::exception& e) {
BOOST_LOG_TRIVIAL(error) << e.what();
// Rethrow the exception
@ -291,13 +297,13 @@ public:
std::fstream out(loc, std::fstream::out | std::fstream::binary);
if(out.good()) {
m_layers_rst[i].first.save(out, Raster::Compression::PNG);
m_layers_rst[i].raster.save(out, Raster::Compression::PNG);
} else {
BOOST_LOG_TRIVIAL(error) << "Can't create file for layer";
}
out.close();
m_layers_rst[i].first.reset();
m_layers_rst[i].raster.reset();
}
void set_statistics(const std::vector<double> statistics)

View file

@ -2,8 +2,6 @@
#include <ExPolygon.hpp>
#include <libnest2d/backends/clipper/clipper_polygon.hpp>
#include <cstdint>
// For rasterizing
#include <agg/agg_basics.h>
#include <agg/agg_rendering_buffer.h>
@ -16,8 +14,8 @@
#include <agg/agg_rasterizer_scanline_aa.h>
#include <agg/agg_path_storage.h>
// For png compression
#include <png/writer.hpp>
// Experimental minz image write:
#include <miniz/miniz_tdef.h>
namespace Slic3r {
@ -201,29 +199,23 @@ void Raster::draw(const ClipperLib::Polygon &poly)
void Raster::save(std::ostream& stream, Compression comp)
{
assert(m_impl);
if(!stream.good()) return;
switch(comp) {
case Compression::PNG: {
png::writer<std::ostream> wr(stream);
wr.set_bit_depth(8);
wr.set_color_type(png::color_type_gray);
wr.set_width(resolution().width_px);
wr.set_height(resolution().height_px);
wr.set_compression_type(png::compression_type_default);
wr.write_info();
auto& b = m_impl->buffer();
auto ptr = reinterpret_cast<png::byte*>( b.data() );
unsigned stride =
sizeof(Impl::TBuffer::value_type) * resolution().width_px;
size_t out_len = 0;
void * rawdata = tdefl_write_image_to_png_file_in_memory(
b.data(),
int(resolution().width_px),
int(resolution().height_px), 1, &out_len);
for(unsigned r = 0; r < resolution().height_px; r++, ptr+=stride) {
wr.write_row(ptr);
}
if(rawdata == nullptr) break;
wr.write_end_info();
stream.write(static_cast<const char*>(rawdata),
std::streamsize(out_len));
MZ_FREE(rawdata);
break;
}
@ -240,4 +232,47 @@ void Raster::save(std::ostream& stream, Compression comp)
}
}
RawBytes Raster::save(Raster::Compression comp)
{
assert(m_impl);
std::uint8_t *ptr = nullptr; size_t s = 0;
switch(comp) {
case Compression::PNG: {
void *rawdata = tdefl_write_image_to_png_file_in_memory(
m_impl->buffer().data(),
int(resolution().width_px),
int(resolution().height_px), 1, &s);
if(rawdata == nullptr) break;
ptr = static_cast<std::uint8_t*>(rawdata);
break;
}
case Compression::RAW: {
auto header = std::string("P5 ") +
std::to_string(m_impl->resolution().width_px) + " " +
std::to_string(m_impl->resolution().height_px) + " " + "255 ";
auto sz = m_impl->buffer().size()*sizeof(Impl::TBuffer::value_type);
s = sz + header.size();
ptr = static_cast<std::uint8_t*>(MZ_MALLOC(s));
auto buff = reinterpret_cast<std::uint8_t*>(m_impl->buffer().data());
std::copy(buff, buff+sz, ptr + header.size());
}
}
return {ptr, s};
}
void RawBytes::MinzDeleter::operator()(uint8_t *rawptr)
{
MZ_FREE(rawptr);
}
}

View file

@ -3,6 +3,8 @@
#include <ostream>
#include <memory>
#include <vector>
#include <cstdint>
namespace ClipperLib { class Polygon; }
@ -10,6 +12,43 @@ namespace Slic3r {
class ExPolygon;
// Raw byte buffer paired with its size. Suitable for compressed PNG data.
class RawBytes {
class MinzDeleter {
public:
void operator()(std::uint8_t *rawptr);
};
std::unique_ptr<std::uint8_t, MinzDeleter> m_buffer = nullptr;
size_t m_size = 0;
public:
RawBytes() = default;
RawBytes(std::uint8_t *rawptr, size_t s): m_buffer(rawptr), m_size(s) {}
size_t size() const { return m_size; }
const uint8_t * data() { return m_buffer.get(); }
// /////////////////////////////////////////////////////////////////////////
// FIXME: the following is needed for MSVC2013 compatibility
// /////////////////////////////////////////////////////////////////////////
RawBytes(const RawBytes&) = delete;
RawBytes(RawBytes&& mv):
m_buffer(std::move(mv.m_buffer)), m_size(mv.m_size) {}
RawBytes& operator=(const RawBytes&) = delete;
RawBytes& operator=(RawBytes&& mv) {
m_buffer.swap(mv.m_buffer);
m_size = mv.m_size;
return *this;
}
// /////////////////////////////////////////////////////////////////////////
};
/**
* @brief Raster captures an anti-aliased monochrome canvas where vectorial
* polygons can be rasterized. Fill color is always white and the background is
@ -90,6 +129,8 @@ public:
/// Save the raster on the specified stream.
void save(std::ostream& stream, Compression comp = Compression::RAW);
RawBytes save(Compression comp = Compression::RAW);
};
}

View file

@ -7,7 +7,7 @@
#include "Point.hpp"
#include "MTUtils.hpp"
#include <libnest2d/backends/clipper/clipper_polygon.hpp>
#include <iterator>
#include "Zipper.hpp"
namespace Slic3r {
@ -316,6 +316,43 @@ struct SLAPrintStatistics
}
};
struct SLAminzZipper {};
// The implementation of creating zipped archives with wxWidgets
template<> class LayerWriter<SLAminzZipper> {
Zipper m_zip;
public:
LayerWriter(const std::string& zipfile_path): m_zip(zipfile_path) {}
void next_entry(const std::string& fname) { m_zip.add_entry(fname); }
void binary_entry(const std::string& fname,
const std::uint8_t* buf,
size_t l)
{
m_zip.add_entry(fname, buf, l);
}
std::string get_name() const {
return m_zip.get_name();
}
template<class T> inline LayerWriter& operator<<(T&& arg) {
m_zip << std::forward<T>(arg); return *this;
}
bool is_ok() const {
return true; // m_zip blows up if something goes wrong...
}
// After finalize, no writing to the archive will have an effect. The only
// valid operation is to dispose the object calling the destructor which
// should close the file. This method can throw and signal potential errors
// when flushing the archive. This is why its present.
void finalize() { m_zip.finalize(); }
};
/**
* @brief This class is the high level FSM for the SLA printing process.
*
@ -348,9 +385,11 @@ public:
// Returns true if the last step was finished with success.
bool finished() const override { return this->is_step_done(slaposSliceSupports) && this->Inherited::is_step_done(slapsRasterize); }
template<class Fmt> void export_raster(const std::string& fname) {
template<class Fmt = SLAminzZipper>
void export_raster(const std::string& fname) {
if(m_printer) m_printer->save<Fmt>(fname);
}
const PrintObjects& objects() const { return m_objects; }
const SLAPrintConfig& print_config() const { return m_print_config; }

228
src/libslic3r/Zipper.cpp Normal file
View file

@ -0,0 +1,228 @@
#include <exception>
#include <sstream>
#include <iostream>
#include "Zipper.hpp"
#include "miniz/miniz_zip.h"
#include <boost/filesystem/path.hpp>
#include <boost/log/trivial.hpp>
#include "I18N.hpp"
//! macro used to mark string used at localization,
//! return same string
#define L(s) Slic3r::I18N::translate(s)
#if defined(_MSC_VER) && _MSC_VER <= 1800 || __cplusplus < 201103L
#define SLIC3R_NORETURN
#elif __cplusplus >= 201103L
#define SLIC3R_NORETURN [[noreturn]]
#endif
namespace Slic3r {
class Zipper::Impl {
public:
mz_zip_archive arch;
std::string m_zipname;
static std::string get_errorstr(mz_zip_error mz_err)
{
switch (mz_err)
{
case MZ_ZIP_NO_ERROR:
return "no error";
case MZ_ZIP_UNDEFINED_ERROR:
return L("undefined error");
case MZ_ZIP_TOO_MANY_FILES:
return L("too many files");
case MZ_ZIP_FILE_TOO_LARGE:
return L("file too large");
case MZ_ZIP_UNSUPPORTED_METHOD:
return L("unsupported method");
case MZ_ZIP_UNSUPPORTED_ENCRYPTION:
return L("unsupported encryption");
case MZ_ZIP_UNSUPPORTED_FEATURE:
return L("unsupported feature");
case MZ_ZIP_FAILED_FINDING_CENTRAL_DIR:
return L("failed finding central directory");
case MZ_ZIP_NOT_AN_ARCHIVE:
return L("not a ZIP archive");
case MZ_ZIP_INVALID_HEADER_OR_CORRUPTED:
return L("invalid header or archive is corrupted");
case MZ_ZIP_UNSUPPORTED_MULTIDISK:
return L("unsupported multidisk archive");
case MZ_ZIP_DECOMPRESSION_FAILED:
return L("decompression failed or archive is corrupted");
case MZ_ZIP_COMPRESSION_FAILED:
return L("compression failed");
case MZ_ZIP_UNEXPECTED_DECOMPRESSED_SIZE:
return L("unexpected decompressed size");
case MZ_ZIP_CRC_CHECK_FAILED:
return L("CRC-32 check failed");
case MZ_ZIP_UNSUPPORTED_CDIR_SIZE:
return L("unsupported central directory size");
case MZ_ZIP_ALLOC_FAILED:
return L("allocation failed");
case MZ_ZIP_FILE_OPEN_FAILED:
return L("file open failed");
case MZ_ZIP_FILE_CREATE_FAILED:
return L("file create failed");
case MZ_ZIP_FILE_WRITE_FAILED:
return L("file write failed");
case MZ_ZIP_FILE_READ_FAILED:
return L("file read failed");
case MZ_ZIP_FILE_CLOSE_FAILED:
return L("file close failed");
case MZ_ZIP_FILE_SEEK_FAILED:
return L("file seek failed");
case MZ_ZIP_FILE_STAT_FAILED:
return L("file stat failed");
case MZ_ZIP_INVALID_PARAMETER:
return L("invalid parameter");
case MZ_ZIP_INVALID_FILENAME:
return L("invalid filename");
case MZ_ZIP_BUF_TOO_SMALL:
return L("buffer too small");
case MZ_ZIP_INTERNAL_ERROR:
return L("internal error");
case MZ_ZIP_FILE_NOT_FOUND:
return L("file not found");
case MZ_ZIP_ARCHIVE_TOO_LARGE:
return L("archive is too large");
case MZ_ZIP_VALIDATION_FAILED:
return L("validation failed");
case MZ_ZIP_WRITE_CALLBACK_FAILED:
return L("write calledback failed");
default:
break;
}
return "unknown error";
}
std::string formatted_errorstr() const
{
return L("Error with zip archive") + " " + m_zipname + ": " +
get_errorstr(arch.m_last_error) + "!";
}
SLIC3R_NORETURN void blow_up() const
{
throw std::runtime_error(formatted_errorstr());
}
bool is_alive()
{
return arch.m_zip_mode != MZ_ZIP_MODE_WRITING_HAS_BEEN_FINALIZED;
}
};
Zipper::Zipper(const std::string &zipfname, e_compression compression)
{
m_impl.reset(new Impl());
m_compression = compression;
m_impl->m_zipname = zipfname;
memset(&m_impl->arch, 0, sizeof(m_impl->arch));
// Initialize the archive data
if(!mz_zip_writer_init_file(&m_impl->arch, zipfname.c_str(), 0))
m_impl->blow_up();
}
Zipper::~Zipper()
{
if(m_impl->is_alive()) {
// Flush the current entry if not finished yet.
try { finish_entry(); } catch(...) {
BOOST_LOG_TRIVIAL(error) << m_impl->formatted_errorstr();
}
if(!mz_zip_writer_finalize_archive(&m_impl->arch))
BOOST_LOG_TRIVIAL(error) << m_impl->formatted_errorstr();
}
// The file should be closed no matter what...
if(!mz_zip_writer_end(&m_impl->arch))
BOOST_LOG_TRIVIAL(error) << m_impl->formatted_errorstr();
}
Zipper::Zipper(Zipper &&m):
m_impl(std::move(m.m_impl)),
m_data(std::move(m.m_data)),
m_entry(std::move(m.m_entry)),
m_compression(m.m_compression) {}
Zipper &Zipper::operator=(Zipper &&m) {
m_impl = std::move(m.m_impl);
m_data = std::move(m.m_data);
m_entry = std::move(m.m_entry);
m_compression = m.m_compression;
return *this;
}
void Zipper::add_entry(const std::string &name)
{
if(!m_impl->is_alive()) return;
finish_entry(); // finish previous business
m_entry = name;
}
void Zipper::add_entry(const std::string &name, const uint8_t *data, size_t l)
{
if(!m_impl->is_alive()) return;
finish_entry();
mz_uint cmpr = MZ_NO_COMPRESSION;
switch (m_compression) {
case NO_COMPRESSION: cmpr = MZ_NO_COMPRESSION; break;
case FAST_COMPRESSION: cmpr = MZ_BEST_SPEED; break;
case TIGHT_COMPRESSION: cmpr = MZ_BEST_COMPRESSION; break;
}
if(!mz_zip_writer_add_mem(&m_impl->arch, name.c_str(), data, l, cmpr))
m_impl->blow_up();
m_entry.clear();
m_data.clear();
}
void Zipper::finish_entry()
{
if(!m_impl->is_alive()) return;
if(!m_data.empty() && !m_entry.empty()) {
mz_uint compression = MZ_NO_COMPRESSION;
switch (m_compression) {
case NO_COMPRESSION: compression = MZ_NO_COMPRESSION; break;
case FAST_COMPRESSION: compression = MZ_BEST_SPEED; break;
case TIGHT_COMPRESSION: compression = MZ_BEST_COMPRESSION; break;
}
if(!mz_zip_writer_add_mem(&m_impl->arch, m_entry.c_str(),
m_data.c_str(),
m_data.size(),
compression)) m_impl->blow_up();
}
m_data.clear();
m_entry.clear();
}
std::string Zipper::get_name() const {
return boost::filesystem::path(m_impl->m_zipname).stem().string();
}
void Zipper::finalize()
{
finish_entry();
if(m_impl->is_alive()) if(!mz_zip_writer_finalize_archive(&m_impl->arch))
m_impl->blow_up();
}
}

93
src/libslic3r/Zipper.hpp Normal file
View file

@ -0,0 +1,93 @@
#ifndef ZIPPER_HPP
#define ZIPPER_HPP
#include <string>
#include <memory>
namespace Slic3r {
// Class for creating zip archives.
class Zipper {
public:
// Three compression levels supported
enum e_compression {
NO_COMPRESSION,
FAST_COMPRESSION,
TIGHT_COMPRESSION
};
private:
class Impl;
std::unique_ptr<Impl> m_impl;
std::string m_data;
std::string m_entry;
e_compression m_compression;
public:
// Will blow up in a runtime exception if the file cannot be created.
explicit Zipper(const std::string& zipfname,
e_compression level = NO_COMPRESSION);
~Zipper();
// No copies allwed, this is a file resource...
Zipper(const Zipper&) = delete;
Zipper& operator=(const Zipper&) = delete;
// Moving is fine.
// Zipper(Zipper&&) = default;
// Zipper& operator=(Zipper&&) = default;
// All becouse of VS2013:
Zipper(Zipper &&m);
Zipper& operator=(Zipper &&m);
/// Adding an entry means a file inside the new archive. Name param is the
/// name of the new file. To create directories, append a forward slash.
/// The previous entry is finished (see finish_entry)
void add_entry(const std::string& name);
/// Add a new binary file entry with an instantly given byte buffer.
/// This method throws exactly like finish_entry() does.
void add_entry(const std::string& name, const std::uint8_t* data, size_t l);
// Writing data to the archive works like with standard streams. The target
// within the zip file is the entry created with the add_entry method.
// Template taking only arithmetic values, that std::to_string can handle.
template<class T> inline
typename std::enable_if<std::is_arithmetic<T>::value, Zipper&>::type
operator<<(T &&val) {
return this->operator<<(std::to_string(std::forward<T>(val)));
}
// Template applied only for types that std::string can handle for append
// and copy. This includes c style strings...
template<class T> inline
typename std::enable_if<!std::is_arithmetic<T>::value, Zipper&>::type
operator<<(T &&val) {
if(m_data.empty()) m_data = std::forward<T>(val);
else m_data.append(val);
return *this;
}
/// Finishing an entry means that subsequent writes will no longer be
/// appended to the previous entry. They will be written into the internal
/// buffer and ones an entry is added, the buffer will bind to the new entry
/// If the buffer was written, but no entry was added, the buffer will be
/// cleared after this call.
///
/// This method will throw a runtime exception if an error occures. The
/// entry will still be open (with the data intact) but the state of the
/// file is up to minz after the erroneous write.
void finish_entry();
/// Gets the name of the archive without the path or extension.
std::string get_name() const;
void finalize();
};
}
#endif // ZIPPER_HPP