OpenVDB  9.0.1
Compression.h
Go to the documentation of this file.
1 // Copyright Contributors to the OpenVDB Project
2 // SPDX-License-Identifier: MPL-2.0
3 
4 #ifndef OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
5 #define OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
6 
7 #include <openvdb/Types.h>
8 #include <openvdb/MetaMap.h>
9 #include <openvdb/math/Math.h> // for negative()
10 #include "io.h" // for getDataCompression(), etc.
11 #include "DelayedLoadMetadata.h"
12 #include <algorithm>
13 #include <iostream>
14 #include <memory>
15 #include <string>
16 #include <vector>
17 
18 
19 namespace openvdb {
21 namespace OPENVDB_VERSION_NAME {
22 namespace io {
23 
24 /// @brief OR-able bit flags for compression options on input and output streams
25 /// @details
26 /// <dl>
27 /// <dt><tt>COMPRESS_NONE</tt>
28 /// <dd>On write, don't compress data.<br>
29 /// On read, the input stream contains uncompressed data.
30 ///
31 /// <dt><tt>COMPRESS_ZIP</tt>
32 /// <dd>When writing grids other than level sets or fog volumes, apply
33 /// ZLIB compression to internal and leaf node value buffers.<br>
34 /// When reading grids other than level sets or fog volumes, indicate that
35 /// the value buffers of internal and leaf nodes are ZLIB-compressed.<br>
36 /// ZLIB compresses well but is slow.
37 ///
38 /// <dt><tt>COMPRESS_ACTIVE_MASK</tt>
39 /// <dd>When writing a grid of any class, don't output a node's inactive values
40 /// if it has two or fewer distinct values. Instead, output minimal information
41 /// to permit the lossless reconstruction of inactive values.<br>
42 /// On read, nodes might have been stored without inactive values.
43 /// Where necessary, reconstruct inactive values from available information.
44 ///
45 /// <dt><tt>COMPRESS_BLOSC</tt>
46 /// <dd>When writing grids other than level sets or fog volumes, apply
47 /// Blosc compression to internal and leaf node value buffers.<br>
48 /// When reading grids other than level sets or fog volumes, indicate that
49 /// the value buffers of internal and leaf nodes are Blosc-compressed.<br>
50 /// Blosc is much faster than ZLIB and produces comparable file sizes.
51 /// </dl>
52 enum {
54  COMPRESS_ZIP = 0x1,
57 };
58 
59 /// Return a string describing the given compression flags.
60 OPENVDB_API std::string compressionToString(uint32_t flags);
61 
62 
63 ////////////////////////////////////////
64 
65 
66 /// @internal Per-node indicator byte that specifies what additional metadata
67 /// is stored to permit reconstruction of inactive values
68 enum {
69  /*0*/ NO_MASK_OR_INACTIVE_VALS, // no inactive vals, or all inactive vals are +background
70  /*1*/ NO_MASK_AND_MINUS_BG, // all inactive vals are -background
71  /*2*/ NO_MASK_AND_ONE_INACTIVE_VAL, // all inactive vals have the same non-background val
72  /*3*/ MASK_AND_NO_INACTIVE_VALS, // mask selects between -background and +background
73  /*4*/ MASK_AND_ONE_INACTIVE_VAL, // mask selects between backgd and one other inactive val
74  /*5*/ MASK_AND_TWO_INACTIVE_VALS, // mask selects between two non-background inactive vals
75  /*6*/ NO_MASK_AND_ALL_VALS // > 2 inactive vals, so no mask compression at all
76 };
77 
78 
79 template <typename ValueT, typename MaskT>
81 {
82  // Comparison function for values
83  static inline bool eq(const ValueT& a, const ValueT& b) {
84  return math::isExactlyEqual(a, b);
85  }
86 
88  const MaskT& valueMask, const MaskT& childMask,
89  const ValueT* srcBuf, const ValueT& background)
90  {
91  /// @todo Consider all values, not just inactive values?
92  inactiveVal[0] = inactiveVal[1] = background;
93  int numUniqueInactiveVals = 0;
94  for (typename MaskT::OffIterator it = valueMask.beginOff();
95  numUniqueInactiveVals < 3 && it; ++it)
96  {
97  const Index32 idx = it.pos();
98 
99  // Skip inactive values that are actually child node pointers.
100  if (childMask.isOn(idx)) continue;
101 
102  const ValueT& val = srcBuf[idx];
103  const bool unique = !(
104  (numUniqueInactiveVals > 0 && MaskCompress::eq(val, inactiveVal[0])) ||
105  (numUniqueInactiveVals > 1 && MaskCompress::eq(val, inactiveVal[1]))
106  );
107  if (unique) {
108  if (numUniqueInactiveVals < 2) inactiveVal[numUniqueInactiveVals] = val;
109  ++numUniqueInactiveVals;
110  }
111  }
112 
113  metadata = NO_MASK_OR_INACTIVE_VALS;
114 
115  if (numUniqueInactiveVals == 1) {
116  if (!MaskCompress::eq(inactiveVal[0], background)) {
117  if (MaskCompress::eq(inactiveVal[0], math::negative(background))) {
118  metadata = NO_MASK_AND_MINUS_BG;
119  } else {
120  metadata = NO_MASK_AND_ONE_INACTIVE_VAL;
121  }
122  }
123  } else if (numUniqueInactiveVals == 2) {
124  metadata = NO_MASK_OR_INACTIVE_VALS;
125  if (!MaskCompress::eq(inactiveVal[0], background) && !MaskCompress::eq(inactiveVal[1], background)) {
126  // If neither inactive value is equal to the background, both values
127  // need to be saved, along with a mask that selects between them.
128  metadata = MASK_AND_TWO_INACTIVE_VALS;
129 
130  } else if (MaskCompress::eq(inactiveVal[1], background)) {
131  if (MaskCompress::eq(inactiveVal[0], math::negative(background))) {
132  // If the second inactive value is equal to the background and
133  // the first is equal to -background, neither value needs to be saved,
134  // but save a mask that selects between -background and +background.
135  metadata = MASK_AND_NO_INACTIVE_VALS;
136  } else {
137  // If the second inactive value is equal to the background, only
138  // the first value needs to be saved, along with a mask that selects
139  // between it and the background.
140  metadata = MASK_AND_ONE_INACTIVE_VAL;
141  }
142  } else if (MaskCompress::eq(inactiveVal[0], background)) {
143  if (MaskCompress::eq(inactiveVal[1], math::negative(background))) {
144  // If the first inactive value is equal to the background and
145  // the second is equal to -background, neither value needs to be saved,
146  // but save a mask that selects between -background and +background.
147  metadata = MASK_AND_NO_INACTIVE_VALS;
148  std::swap(inactiveVal[0], inactiveVal[1]);
149  } else {
150  // If the first inactive value is equal to the background, swap it
151  // with the second value and save only that value, along with a mask
152  // that selects between it and the background.
153  std::swap(inactiveVal[0], inactiveVal[1]);
154  metadata = MASK_AND_ONE_INACTIVE_VAL;
155  }
156  }
157  } else if (numUniqueInactiveVals > 2) {
158  metadata = NO_MASK_AND_ALL_VALS;
159  }
160  }
161 
162  int8_t metadata = NO_MASK_AND_ALL_VALS;
163  ValueT inactiveVal[2];
164 };
165 
166 
167 ////////////////////////////////////////
168 
169 
170 /// @brief RealToHalf and its specializations define a mapping from
171 /// floating-point data types to analogous half float types.
172 template<typename T>
173 struct RealToHalf {
174  enum { isReal = false }; // unless otherwise specified, type T is not a floating-point type
175  using HalfT = T; // type T's half float analogue is T itself
176  static HalfT convert(const T& val) { return val; }
177 };
178 template<> struct RealToHalf<float> {
179  enum { isReal = true };
180  using HalfT = math::half;
181  static HalfT convert(float val) { return HalfT(val); }
182 };
183 template<> struct RealToHalf<double> {
184  enum { isReal = true };
185  using HalfT = math::half;
186  // A half can only be constructed from a float, so cast the value to a float first.
187  static HalfT convert(double val) { return HalfT(float(val)); }
188 };
189 template<> struct RealToHalf<Vec2s> {
190  enum { isReal = true };
191  using HalfT = Vec2H;
192  static HalfT convert(const Vec2s& val) { return HalfT(val); }
193 };
194 template<> struct RealToHalf<Vec2d> {
195  enum { isReal = true };
196  using HalfT = Vec2H;
197  // A half can only be constructed from a float, so cast the vector's elements to floats first.
198  static HalfT convert(const Vec2d& val) { return HalfT(Vec2s(val)); }
199 };
200 template<> struct RealToHalf<Vec3s> {
201  enum { isReal = true };
202  using HalfT = Vec3H;
203  static HalfT convert(const Vec3s& val) { return HalfT(val); }
204 };
205 template<> struct RealToHalf<Vec3d> {
206  enum { isReal = true };
207  using HalfT = Vec3H;
208  // A half can only be constructed from a float, so cast the vector's elements to floats first.
209  static HalfT convert(const Vec3d& val) { return HalfT(Vec3s(val)); }
210 };
211 
212 
213 /// Return the given value truncated to 16-bit float precision.
214 template<typename T>
215 inline T
216 truncateRealToHalf(const T& val)
217 {
218  return T(RealToHalf<T>::convert(val));
219 }
220 
221 
222 ////////////////////////////////////////
223 
224 
225 OPENVDB_API size_t zipToStreamSize(const char* data, size_t numBytes);
226 OPENVDB_API void zipToStream(std::ostream&, const char* data, size_t numBytes);
227 OPENVDB_API void unzipFromStream(std::istream&, char* data, size_t numBytes);
228 OPENVDB_API size_t bloscToStreamSize(const char* data, size_t valSize, size_t numVals);
229 OPENVDB_API void bloscToStream(std::ostream&, const char* data, size_t valSize, size_t numVals);
230 OPENVDB_API void bloscFromStream(std::istream&, char* data, size_t numBytes);
231 
232 /// @brief Read data from a stream.
233 /// @param is the input stream
234 /// @param data the contiguous array of data to read in
235 /// @param count the number of elements to read in
236 /// @param compression whether and how the data is compressed (either COMPRESS_NONE,
237 /// COMPRESS_ZIP, COMPRESS_ACTIVE_MASK or COMPRESS_BLOSC)
238 /// @param metadata optional pointer to a DelayedLoadMetadata object that stores
239 /// the size of the compressed buffer
240 /// @param metadataOffset offset into DelayedLoadMetadata, ignored if pointer is null
241 /// @throw IoError if @a compression is COMPRESS_BLOSC but OpenVDB was compiled
242 /// without Blosc support.
243 /// @details This default implementation is instantiated only for types
244 /// whose size can be determined by the sizeof() operator.
245 template<typename T>
246 inline void
247 readData(std::istream& is, T* data, Index count, uint32_t compression,
248  DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0))
249 {
250  const bool seek = data == nullptr;
251  if (seek) {
252  assert(!getStreamMetadataPtr(is) || getStreamMetadataPtr(is)->seekable());
253  }
254  const bool hasCompression = compression & (COMPRESS_BLOSC | COMPRESS_ZIP);
255 
256  if (metadata && seek && hasCompression) {
257  size_t compressedSize = metadata->getCompressedSize(metadataOffset);
258  is.seekg(compressedSize, std::ios_base::cur);
259  } else if (compression & COMPRESS_BLOSC) {
260  bloscFromStream(is, reinterpret_cast<char*>(data), sizeof(T) * count);
261  } else if (compression & COMPRESS_ZIP) {
262  unzipFromStream(is, reinterpret_cast<char*>(data), sizeof(T) * count);
263  } else if (seek) {
264  is.seekg(sizeof(T) * count, std::ios_base::cur);
265  } else {
266  is.read(reinterpret_cast<char*>(data), sizeof(T) * count);
267  }
268 }
269 
270 /// Specialization for std::string input
271 template<>
272 inline void
273 readData<std::string>(std::istream& is, std::string* data, Index count, uint32_t /*compression*/,
274  DelayedLoadMetadata* /*metadata*/, size_t /*metadataOffset*/)
275 {
276  for (Index i = 0; i < count; ++i) {
277  size_t len = 0;
278  is >> len;
279  //data[i].resize(len);
280  //is.read(&(data[i][0]), len);
281 
282  std::string buffer(len+1, ' ');
283  is.read(&buffer[0], len+1);
284  if (data != nullptr) data[i].assign(buffer, 0, len);
285  }
286 }
287 
288 /// HalfReader wraps a static function, read(), that is analogous to readData(), above,
289 /// except that it is partially specialized for floating-point types in order to promote
290 /// 16-bit half float values to full float. A wrapper class is required because
291 /// only classes, not functions, can be partially specialized.
292 template<bool IsReal, typename T> struct HalfReader;
293 /// Partial specialization for non-floating-point types (no half to float promotion)
294 template<typename T>
295 struct HalfReader</*IsReal=*/false, T> {
296  static inline void read(std::istream& is, T* data, Index count, uint32_t compression,
297  DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) {
298  readData(is, data, count, compression, metadata, metadataOffset);
299  }
300 };
301 /// Partial specialization for floating-point types
302 template<typename T>
303 struct HalfReader</*IsReal=*/true, T> {
304  using HalfT = typename RealToHalf<T>::HalfT;
305  static inline void read(std::istream& is, T* data, Index count, uint32_t compression,
306  DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) {
307  if (count < 1) return;
308  if (data == nullptr) {
309  // seek mode - pass through null pointer
310  readData<HalfT>(is, nullptr, count, compression, metadata, metadataOffset);
311  } else {
312  std::vector<HalfT> halfData(count); // temp buffer into which to read half float values
313  readData<HalfT>(is, reinterpret_cast<HalfT*>(&halfData[0]), count, compression,
314  metadata, metadataOffset);
315  // Copy half float values from the temporary buffer to the full float output array.
316  std::copy(halfData.begin(), halfData.end(), data);
317  }
318  }
319 };
320 
321 
322 template<typename T>
323 inline size_t
324 writeDataSize(const T *data, Index count, uint32_t compression)
325 {
326  if (compression & COMPRESS_BLOSC) {
327  return bloscToStreamSize(reinterpret_cast<const char*>(data), sizeof(T), count);
328  } else if (compression & COMPRESS_ZIP) {
329  return zipToStreamSize(reinterpret_cast<const char*>(data), sizeof(T) * count);
330  } else {
331  return sizeof(T) * count;
332  }
333 }
334 
335 
336 /// Specialization for std::string output
337 template<>
338 inline size_t
339 writeDataSize<std::string>(const std::string* data, Index count,
340  uint32_t /*compression*/) ///< @todo add compression
341 {
342  size_t size(0);
343  for (Index i = 0; i < count; ++i) {
344  const size_t len = data[i].size();
345  size += sizeof(size_t) + (len+1);
346  }
347  return size;
348 }
349 
350 
351 /// Write data to a stream.
352 /// @param os the output stream
353 /// @param data the contiguous array of data to write
354 /// @param count the number of elements to write out
355 /// @param compression whether and how to compress the data (either COMPRESS_NONE,
356 /// COMPRESS_ZIP, COMPRESS_ACTIVE_MASK or COMPRESS_BLOSC)
357 /// @throw IoError if @a compression is COMPRESS_BLOSC but OpenVDB was compiled
358 /// without Blosc support.
359 /// @details This default implementation is instantiated only for types
360 /// whose size can be determined by the sizeof() operator.
361 template<typename T>
362 inline void
363 writeData(std::ostream &os, const T *data, Index count, uint32_t compression)
364 {
365  if (compression & COMPRESS_BLOSC) {
366  bloscToStream(os, reinterpret_cast<const char*>(data), sizeof(T), count);
367  } else if (compression & COMPRESS_ZIP) {
368  zipToStream(os, reinterpret_cast<const char*>(data), sizeof(T) * count);
369  } else {
370  os.write(reinterpret_cast<const char*>(data), sizeof(T) * count);
371  }
372 }
373 
374 /// Specialization for std::string output
375 template<>
376 inline void
377 writeData<std::string>(std::ostream& os, const std::string* data, Index count,
378  uint32_t /*compression*/) ///< @todo add compression
379 {
380  for (Index i = 0; i < count; ++i) {
381  const size_t len = data[i].size();
382  os << len;
383  os.write(data[i].c_str(), len+1);
384  //os.write(&(data[i][0]), len );
385  }
386 }
387 
388 /// HalfWriter wraps a static function, write(), that is analogous to writeData(), above,
389 /// except that it is partially specialized for floating-point types in order to quantize
390 /// floating-point values to 16-bit half float. A wrapper class is required because
391 /// only classes, not functions, can be partially specialized.
392 template<bool IsReal, typename T> struct HalfWriter;
393 /// Partial specialization for non-floating-point types (no float to half quantization)
394 template<typename T>
395 struct HalfWriter</*IsReal=*/false, T> {
396  static inline size_t writeSize(const T* data, Index count, uint32_t compression) {
397  return writeDataSize(data, count, compression);
398  }
399  static inline void write(std::ostream& os, const T* data, Index count, uint32_t compression) {
400  writeData(os, data, count, compression);
401  }
402 };
403 /// Partial specialization for floating-point types
404 template<typename T>
405 struct HalfWriter</*IsReal=*/true, T> {
406  using HalfT = typename RealToHalf<T>::HalfT;
407  static inline size_t writeSize(const T* data, Index count, uint32_t compression) {
408  if (count < 1) return size_t(0);
409  // Convert full float values to half float, then output the half float array.
410  std::vector<HalfT> halfData(count);
411  for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<T>::convert(data[i]);
412  return writeDataSize<HalfT>(reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
413  }
414  static inline void write(std::ostream& os, const T* data, Index count, uint32_t compression) {
415  if (count < 1) return;
416  // Convert full float values to half float, then output the half float array.
417  std::vector<HalfT> halfData(count);
418  for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<T>::convert(data[i]);
419  writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
420  }
421 };
422 #ifdef _MSC_VER
423 /// Specialization to avoid double to float warnings in MSVC
424 template<>
425 struct HalfWriter</*IsReal=*/true, double> {
427  static inline size_t writeSize(const double* data, Index count, uint32_t compression)
428  {
429  if (count < 1) return size_t(0);
430  // Convert full float values to half float, then output the half float array.
431  std::vector<HalfT> halfData(count);
432  for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<double>::convert(data[i]);
433  return writeDataSize<HalfT>(reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
434  }
435  static inline void write(std::ostream& os, const double* data, Index count,
436  uint32_t compression)
437  {
438  if (count < 1) return;
439  // Convert full float values to half float, then output the half float array.
440  std::vector<HalfT> halfData(count);
441  for (Index i = 0; i < count; ++i) halfData[i] = RealToHalf<double>::convert(data[i]);
442  writeData<HalfT>(os, reinterpret_cast<const HalfT*>(&halfData[0]), count, compression);
443  }
444 };
445 #endif // _MSC_VER
446 
447 
448 ////////////////////////////////////////
449 
450 
451 /// Populate the given buffer with @a destCount values of type @c ValueT
452 /// read from the given stream, taking into account that the stream might
453 /// have been compressed via one of several supported schemes.
454 /// [Mainly for internal use]
455 /// @param is a stream from which to read data (possibly compressed,
456 /// depending on the stream's compression settings)
457 /// @param destBuf a buffer into which to read values of type @c ValueT
458 /// @param destCount the number of values to be stored in the buffer
459 /// @param valueMask a bitmask (typically, a node's value mask) indicating
460 /// which positions in the buffer correspond to active values
461 /// @param fromHalf if true, read 16-bit half floats from the input stream
462 /// and convert them to full floats
463 template<typename ValueT, typename MaskT>
464 inline void
465 readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount,
466  const MaskT& valueMask, bool fromHalf)
467 {
468  // Get the stream's compression settings.
469  auto meta = getStreamMetadataPtr(is);
470  const uint32_t compression = getDataCompression(is);
471  const bool maskCompressed = compression & COMPRESS_ACTIVE_MASK;
472 
473  const bool seek = (destBuf == nullptr);
474  assert(!seek || (!meta || meta->seekable()));
475 
476  // Get delayed load metadata if it exists
477 
478  DelayedLoadMetadata::Ptr delayLoadMeta;
479  uint64_t leafIndex(0);
480  if (seek && meta && meta->delayedLoadMeta()) {
481  delayLoadMeta =
482  meta->gridMetadata().getMetadata<DelayedLoadMetadata>("file_delayed_load");
483  leafIndex = meta->leaf();
484  }
485 
486  int8_t metadata = NO_MASK_AND_ALL_VALS;
488  // Read the flag that specifies what, if any, additional metadata
489  // (selection mask and/or inactive value(s)) is saved.
490  if (seek && !maskCompressed) {
491  is.seekg(/*bytes=*/1, std::ios_base::cur);
492  } else if (seek && delayLoadMeta) {
493  metadata = delayLoadMeta->getMask(leafIndex);
494  is.seekg(/*bytes=*/1, std::ios_base::cur);
495  } else {
496  is.read(reinterpret_cast<char*>(&metadata), /*bytes=*/1);
497  }
498  }
499 
500  ValueT background = zeroVal<ValueT>();
501  if (const void* bgPtr = getGridBackgroundValuePtr(is)) {
502  background = *static_cast<const ValueT*>(bgPtr);
503  }
504  ValueT inactiveVal1 = background;
505  ValueT inactiveVal0 =
506  ((metadata == NO_MASK_OR_INACTIVE_VALS) ? background : math::negative(background));
507 
508  if (metadata == NO_MASK_AND_ONE_INACTIVE_VAL ||
509  metadata == MASK_AND_ONE_INACTIVE_VAL ||
510  metadata == MASK_AND_TWO_INACTIVE_VALS)
511  {
512  // Read one of at most two distinct inactive values.
513  if (seek) {
514  is.seekg(/*bytes=*/sizeof(ValueT), std::ios_base::cur);
515  } else {
516  is.read(reinterpret_cast<char*>(&inactiveVal0), /*bytes=*/sizeof(ValueT));
517  }
518  if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
519  // Read the second of two distinct inactive values.
520  if (seek) {
521  is.seekg(/*bytes=*/sizeof(ValueT), std::ios_base::cur);
522  } else {
523  is.read(reinterpret_cast<char*>(&inactiveVal1), /*bytes=*/sizeof(ValueT));
524  }
525  }
526  }
527 
528  MaskT selectionMask;
529  if (metadata == MASK_AND_NO_INACTIVE_VALS ||
530  metadata == MASK_AND_ONE_INACTIVE_VAL ||
531  metadata == MASK_AND_TWO_INACTIVE_VALS)
532  {
533  // For use in mask compression (only), read the bitmask that selects
534  // between two distinct inactive values.
535  if (seek) {
536  is.seekg(/*bytes=*/selectionMask.memUsage(), std::ios_base::cur);
537  } else {
538  selectionMask.load(is);
539  }
540  }
541 
542  ValueT* tempBuf = destBuf;
543  std::unique_ptr<ValueT[]> scopedTempBuf;
544 
545  Index tempCount = destCount;
546 
547  if (maskCompressed && metadata != NO_MASK_AND_ALL_VALS
549  {
550  tempCount = valueMask.countOn();
551  if (!seek && tempCount != destCount) {
552  // If this node has inactive voxels, allocate a temporary buffer
553  // into which to read just the active values.
554  scopedTempBuf.reset(new ValueT[tempCount]);
555  tempBuf = scopedTempBuf.get();
556  }
557  }
558 
559  // Read in the buffer.
560  if (fromHalf) {
562  is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
563  } else {
564  readData<ValueT>(
565  is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
566  }
567 
568  // If mask compression is enabled and the number of active values read into
569  // the temp buffer is smaller than the size of the destination buffer,
570  // then there are missing (inactive) values.
571  if (!seek && maskCompressed && tempCount != destCount) {
572  // Restore inactive values, using the background value and, if available,
573  // the inside/outside mask. (For fog volumes, the destination buffer is assumed
574  // to be initialized to background value zero, so inactive values can be ignored.)
575  for (Index destIdx = 0, tempIdx = 0; destIdx < MaskT::SIZE; ++destIdx) {
576  if (valueMask.isOn(destIdx)) {
577  // Copy a saved active value into this node's buffer.
578  destBuf[destIdx] = tempBuf[tempIdx];
579  ++tempIdx;
580  } else {
581  // Reconstruct an unsaved inactive value and copy it into this node's buffer.
582  destBuf[destIdx] = (selectionMask.isOn(destIdx) ? inactiveVal1 : inactiveVal0);
583  }
584  }
585  }
586 }
587 
588 
589 template<typename ValueT, typename MaskT>
590 inline size_t
591 writeCompressedValuesSize(ValueT* srcBuf, Index srcCount,
592  const MaskT& valueMask, uint8_t maskMetadata, bool toHalf, uint32_t compress)
593 {
594  using NonConstValueT = typename std::remove_const<ValueT>::type;
595 
596  const bool maskCompress = compress & COMPRESS_ACTIVE_MASK;
597 
598  Index tempCount = srcCount;
599  ValueT* tempBuf = srcBuf;
600  std::unique_ptr<NonConstValueT[]> scopedTempBuf;
601 
602  if (maskCompress && maskMetadata != NO_MASK_AND_ALL_VALS) {
603 
604  tempCount = 0;
605 
606  Index64 onVoxels = valueMask.countOn();
607  if (onVoxels > Index64(0)) {
608  // Create a new array to hold just the active values.
609  scopedTempBuf.reset(new NonConstValueT[onVoxels]);
610  NonConstValueT* localTempBuf = scopedTempBuf.get();
611 
612  // Copy active values to a new, contiguous array.
613  for (typename MaskT::OnIterator it = valueMask.beginOn(); it; ++it, ++tempCount) {
614  localTempBuf[tempCount] = srcBuf[it.pos()];
615  }
616 
617  tempBuf = scopedTempBuf.get();
618  }
619  }
620 
621  // Return the buffer size.
622  if (toHalf) {
623  return HalfWriter<RealToHalf<NonConstValueT>::isReal, NonConstValueT>::writeSize(
624  tempBuf, tempCount, compress);
625  } else {
626  return writeDataSize<NonConstValueT>(tempBuf, tempCount, compress);
627  }
628 }
629 
630 
631 /// Write @a srcCount values of type @c ValueT to the given stream, optionally
632 /// after compressing the values via one of several supported schemes.
633 /// [Mainly for internal use]
634 /// @param os a stream to which to write data (possibly compressed, depending
635 /// on the stream's compression settings)
636 /// @param srcBuf a buffer containing values of type @c ValueT to be written
637 /// @param srcCount the number of values stored in the buffer
638 /// @param valueMask a bitmask (typically, a node's value mask) indicating
639 /// which positions in the buffer correspond to active values
640 /// @param childMask a bitmask (typically, a node's child mask) indicating
641 /// which positions in the buffer correspond to child node pointers
642 /// @param toHalf if true, convert floating-point values to 16-bit half floats
643 template<typename ValueT, typename MaskT>
644 inline void
645 writeCompressedValues(std::ostream& os, ValueT* srcBuf, Index srcCount,
646  const MaskT& valueMask, const MaskT& childMask, bool toHalf)
647 {
648  // Get the stream's compression settings.
649  const uint32_t compress = getDataCompression(os);
650  const bool maskCompress = compress & COMPRESS_ACTIVE_MASK;
651 
652  Index tempCount = srcCount;
653  ValueT* tempBuf = srcBuf;
654  std::unique_ptr<ValueT[]> scopedTempBuf;
655 
656  int8_t metadata = NO_MASK_AND_ALL_VALS;
657 
658  if (!maskCompress) {
659  os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
660  } else {
661  // A valid level set's inactive values are either +background (outside)
662  // or -background (inside), and a fog volume's inactive values are all zero.
663  // Rather than write out all of these values, we can store just the active values
664  // (given that the value mask specifies their positions) and, if necessary,
665  // an inside/outside bitmask.
666 
667  const ValueT zero = zeroVal<ValueT>();
668  ValueT background = zero;
669  if (const void* bgPtr = getGridBackgroundValuePtr(os)) {
670  background = *static_cast<const ValueT*>(bgPtr);
671  }
672 
673  MaskCompress<ValueT, MaskT> maskCompressData(valueMask, childMask, srcBuf, background);
674  metadata = maskCompressData.metadata;
675 
676  os.write(reinterpret_cast<const char*>(&metadata), /*bytes=*/1);
677 
678  if (metadata == NO_MASK_AND_ONE_INACTIVE_VAL ||
679  metadata == MASK_AND_ONE_INACTIVE_VAL ||
680  metadata == MASK_AND_TWO_INACTIVE_VALS)
681  {
682  if (!toHalf) {
683  // Write one of at most two distinct inactive values.
684  os.write(reinterpret_cast<const char*>(&maskCompressData.inactiveVal[0]), sizeof(ValueT));
685  if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
686  // Write the second of two distinct inactive values.
687  os.write(reinterpret_cast<const char*>(&maskCompressData.inactiveVal[1]), sizeof(ValueT));
688  }
689  } else {
690  // Write one of at most two distinct inactive values.
691  ValueT truncatedVal = static_cast<ValueT>(truncateRealToHalf(maskCompressData.inactiveVal[0]));
692  os.write(reinterpret_cast<const char*>(&truncatedVal), sizeof(ValueT));
693  if (metadata == MASK_AND_TWO_INACTIVE_VALS) {
694  // Write the second of two distinct inactive values.
695  truncatedVal = truncateRealToHalf(maskCompressData.inactiveVal[1]);
696  os.write(reinterpret_cast<const char*>(&truncatedVal), sizeof(ValueT));
697  }
698  }
699  }
700 
701  if (metadata == NO_MASK_AND_ALL_VALS) {
702  // If there are more than two unique inactive values, the entire input buffer
703  // needs to be saved (both active and inactive values).
704  /// @todo Save the selection mask as long as most of the inactive values
705  /// are one of two values?
706  } else {
707  // Create a new array to hold just the active values.
708  scopedTempBuf.reset(new ValueT[srcCount]);
709  tempBuf = scopedTempBuf.get();
710 
711  if (metadata == NO_MASK_OR_INACTIVE_VALS ||
712  metadata == NO_MASK_AND_MINUS_BG ||
713  metadata == NO_MASK_AND_ONE_INACTIVE_VAL)
714  {
715  // Copy active values to the contiguous array.
716  tempCount = 0;
717  for (typename MaskT::OnIterator it = valueMask.beginOn(); it; ++it, ++tempCount) {
718  tempBuf[tempCount] = srcBuf[it.pos()];
719  }
720  } else {
721  // Copy active values to a new, contiguous array and populate a bitmask
722  // that selects between two distinct inactive values.
723  MaskT selectionMask;
724  tempCount = 0;
725  for (Index srcIdx = 0; srcIdx < srcCount; ++srcIdx) {
726  if (valueMask.isOn(srcIdx)) { // active value
727  tempBuf[tempCount] = srcBuf[srcIdx];
728  ++tempCount;
729  } else { // inactive value
730  if (MaskCompress<ValueT, MaskT>::eq(srcBuf[srcIdx], maskCompressData.inactiveVal[1])) {
731  selectionMask.setOn(srcIdx); // inactive value 1
732  } // else inactive value 0
733  }
734  }
735  assert(tempCount == valueMask.countOn());
736 
737  // Write out the mask that selects between two inactive values.
738  selectionMask.save(os);
739  }
740  }
741  }
742 
743  // Write out the buffer.
744  if (toHalf) {
745  HalfWriter<RealToHalf<ValueT>::isReal, ValueT>::write(os, tempBuf, tempCount, compress);
746  } else {
747  writeData(os, tempBuf, tempCount, compress);
748  }
749 }
750 
751 } // namespace io
752 } // namespace OPENVDB_VERSION_NAME
753 } // namespace openvdb
754 
755 #endif // OPENVDB_IO_COMPRESSION_HAS_BEEN_INCLUDED
#define OPENVDB_API
Definition: Platform.h:254
bool isExactlyEqual(const T0 &a, const T1 &b)
Return true if a is exactly equal to b.
Definition: Math.h:444
math::Vec2< math::half > Vec2H
Definition: Types.h:66
General-purpose arithmetic and comparison routines, most of which accept arbitrary value types (or at...
static HalfT convert(double val)
Definition: Compression.h:187
static size_t writeSize(const T *data, Index count, uint32_t compression)
Definition: Compression.h:396
T truncateRealToHalf(const T &val)
Return the given value truncated to 16-bit float precision.
Definition: Compression.h:216
Definition: Vec2.h:23
Vec2< float > Vec2s
Definition: Vec2.h:536
Definition: Compression.h:392
Vec3< double > Vec3d
Definition: Vec3.h:668
typename RealToHalf< T >::HalfT HalfT
Definition: Compression.h:406
Definition: Compression.h:70
static HalfT convert(const Vec3s &val)
Definition: Compression.h:203
size_t writeDataSize(const T *data, Index count, uint32_t compression)
Definition: Compression.h:324
static HalfT convert(const T &val)
Definition: Compression.h:176
OPENVDB_API const void * getGridBackgroundValuePtr(std::ios_base &)
Return a pointer to the background value of the grid currently being read from or written to the give...
void readCompressedValues(std::istream &is, ValueT *destBuf, Index destCount, const MaskT &valueMask, bool fromHalf)
Definition: Compression.h:465
void writeData(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition: Compression.h:363
void readData(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Read data from a stream.
Definition: Compression.h:247
Definition: Compression.h:55
math::Vec3< math::half > Vec3H
Definition: Types.h:75
SharedPtr< DelayedLoadMetadata > Ptr
Definition: DelayedLoadMetadata.h:24
static fileSize_t write(std::ostream &os, const GridHandle< BufferT > &handle, Codec codec)
static void read(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Definition: Compression.h:305
OPENVDB_API SharedPtr< StreamMetadata > getStreamMetadataPtr(std::ios_base &)
Return a shared pointer to an object that stores metadata (file format, compression scheme...
OPENVDB_API uint32_t getFormatVersion(std::ios_base &)
Return the file format version number associated with the given input stream.
typename RealToHalf< T >::HalfT HalfT
Definition: Compression.h:304
static void write(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition: Compression.h:414
static void read(std::istream &is, GridHandle< BufferT > &handle, Codec codec)
static void write(std::ostream &os, const T *data, Index count, uint32_t compression)
Definition: Compression.h:399
static HalfT convert(const Vec2s &val)
Definition: Compression.h:192
Definition: Compression.h:53
void writeCompressedValues(std::ostream &os, ValueT *srcBuf, Index srcCount, const MaskT &valueMask, const MaskT &childMask, bool toHalf)
Definition: Compression.h:645
Definition: Compression.h:292
OPENVDB_API void zipToStream(std::ostream &, const char *data, size_t numBytes)
T HalfT
Definition: Compression.h:175
static bool eq(const ValueT &a, const ValueT &b)
Definition: Compression.h:83
static size_t writeSize(const T *data, Index count, uint32_t compression)
Definition: Compression.h:407
OPENVDB_API void bloscToStream(std::ostream &, const char *data, size_t valSize, size_t numVals)
uint64_t Index64
Definition: Types.h:53
Vec2< double > Vec2d
Definition: Vec2.h:537
Definition: Exceptions.h:13
OPENVDB_API std::string compressionToString(uint32_t flags)
Return a string describing the given compression flags.
size_t writeCompressedValuesSize(ValueT *srcBuf, Index srcCount, const MaskT &valueMask, uint8_t maskMetadata, bool toHalf, uint32_t compress)
Definition: Compression.h:591
Index32 Index
Definition: Types.h:54
RealToHalf and its specializations define a mapping from floating-point data types to analogous half ...
Definition: Compression.h:173
static HalfT convert(float val)
Definition: Compression.h:181
OPENVDB_API size_t bloscToStreamSize(const char *data, size_t valSize, size_t numVals)
MaskCompress(const MaskT &valueMask, const MaskT &childMask, const ValueT *srcBuf, const ValueT &background)
Definition: Compression.h:87
static void read(std::istream &is, T *data, Index count, uint32_t compression, DelayedLoadMetadata *metadata=nullptr, size_t metadataOffset=size_t(0))
Definition: Compression.h:296
Definition: Compression.h:56
Definition: Mat.h:187
T negative(const T &val)
Return the unary negation of the given value.
Definition: Math.h:127
static HalfT convert(const Vec2d &val)
Definition: Compression.h:198
Definition: Compression.h:80
Store a buffer of data that can be optionally used during reading for faster delayed-load I/O perform...
Definition: DelayedLoadMetadata.h:21
static HalfT convert(const Vec3d &val)
Definition: Compression.h:209
OPENVDB_API void bloscFromStream(std::istream &, char *data, size_t numBytes)
Vec3< float > Vec3s
Definition: Vec3.h:667
Definition: Compression.h:54
OPENVDB_API void unzipFromStream(std::istream &, char *data, size_t numBytes)
Definition: Compression.h:75
OPENVDB_API size_t zipToStreamSize(const char *data, size_t numBytes)
uint32_t Index32
Definition: Types.h:52
OPENVDB_API uint32_t getDataCompression(std::ios_base &)
Return a bitwise OR of compression option flags (COMPRESS_ZIP, COMPRESS_ACTIVE_MASK, etc.) specifying whether and how input data is compressed or output data should be compressed.
#define OPENVDB_VERSION_NAME
The version namespace name for this library version.
Definition: version.h.in:116
ValueT inactiveVal[2]
Definition: Compression.h:163
internal::half half
Definition: Types.h:29
#define OPENVDB_USE_VERSION_NAMESPACE
Definition: version.h.in:202
int8_t metadata
Definition: Compression.h:162