Add pluggable compression

Author: lucagiac81

db/blob/blob_file_builder.cc

@@ -66,7 +66,7 @@ BlobFileBuilder::BlobFileBuilder(
       immutable_options_(immutable_options),
       min_blob_size_(mutable_cf_options->min_blob_size),
       blob_file_size_(mutable_cf_options->blob_file_size),
-      blob_compressor_(mutable_cf_options->blob_compressor),
+      blob_compressor_(mutable_cf_options->derived_blob_compressor),
       prepopulate_blob_cache_(mutable_cf_options->prepopulate_blob_cache),
       file_options_(file_options),
       write_options_(write_options),

db/blob/blob_file_builder.h

@@ -12,11 +12,11 @@
 
 #include "rocksdb/advanced_options.h"
 #include "rocksdb/compression_type.h"
+#include "rocksdb/compressor.h"
 #include "rocksdb/env.h"
 #include "rocksdb/options.h"
 #include "rocksdb/rocksdb_namespace.h"
 #include "rocksdb/types.h"
-#include "util/compressor.h"
 
 namespace ROCKSDB_NAMESPACE {
 

db/blob/blob_file_reader.h

@@ -13,7 +13,6 @@
 #include "rocksdb/compression_type.h"
 #include "rocksdb/rocksdb_namespace.h"
 #include "util/autovector.h"
-#include "util/compressor.h"
 
 namespace ROCKSDB_NAMESPACE {
 

db/blob/blob_read_request.h

@@ -8,10 +8,10 @@
 #include <cinttypes>
 
 #include "rocksdb/compression_type.h"
+#include "rocksdb/compressor.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/status.h"
 #include "util/autovector.h"
-#include "util/compressor.h"
 
 namespace ROCKSDB_NAMESPACE {
 

db/blob/blob_source.h

@@ -14,10 +14,10 @@
 #include "db/blob/blob_file_cache.h"
 #include "db/blob/blob_read_request.h"
 #include "rocksdb/cache.h"
+#include "rocksdb/compressor.h"
 #include "rocksdb/rocksdb_namespace.h"
 #include "table/block_based/cachable_entry.h"
 #include "util/autovector.h"
-#include "util/compressor.h"
 
 namespace ROCKSDB_NAMESPACE {
 

db/column_family.cc

@@ -113,17 +113,18 @@ void GetIntTblPropCollectorFactory(
 Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options) {
   MutableCFOptions moptions(cf_options);
   ImmutableCFOptions ioptions(cf_options);
-  if (moptions.compressor && !moptions.compressor->Supported()) {
+  if (moptions.derived_compressor &&
+      !moptions.derived_compressor->Supported()) {
     return Status::InvalidArgument("Compression type " +
-                                   moptions.compressor->GetId() +
+                                   moptions.derived_compressor->GetId() +
                                    " is not linked with the binary.");
-  } else if (moptions.bottommost_compressor &&
-             !moptions.bottommost_compressor->Supported()) {
-    return Status::InvalidArgument("Compression type " +
-                                   moptions.bottommost_compressor->GetId() +
-                                   " is not linked with the binary.");
-  } else if (!moptions.compressor_per_level.empty()) {
-    for (const auto& compressor : moptions.compressor_per_level) {
+  } else if (moptions.derived_bottommost_compressor &&
+             !moptions.derived_bottommost_compressor->Supported()) {
+    return Status::InvalidArgument(
+        "Compression type " + moptions.derived_bottommost_compressor->GetId() +
+        " is not linked with the binary.");
+  } else if (!moptions.derived_compressor_per_level.empty()) {
+    for (const auto& compressor : moptions.derived_compressor_per_level) {
       if (compressor == nullptr) {
         return Status::InvalidArgument("Compression type is invalid.");
       } else if (!compressor->Supported()) {
@@ -153,9 +154,10 @@ Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options) {
           "should be nonzero if we're using zstd's dictionary generator.");
     }
   }
-  if (moptions.blob_compressor && !moptions.blob_compressor->Supported()) {
+  if (moptions.derived_blob_compressor &&
+      !moptions.derived_blob_compressor->Supported()) {
     return Status::InvalidArgument("Blob compression type " +
-                                   moptions.blob_compressor->GetId() +
+                                   moptions.derived_blob_compressor->GetId() +
                                    " is not linked with the binary.");
   }
   return Status::OK();

db/compact_files_test.cc

@@ -11,12 +11,12 @@
 
 #include "db/db_impl/db_impl.h"
 #include "port/port.h"
+#include "rocksdb/compressor.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "test_util/sync_point.h"
 #include "test_util/testharness.h"
 #include "util/cast_util.h"
-#include "util/compressor.h"
 #include "util/string_util.h"
 
 namespace ROCKSDB_NAMESPACE {

db/compaction/compaction_picker.cc

@@ -87,12 +87,12 @@ std::shared_ptr<Compressor> GetCompressor(const VersionStorageInfo* vstorage,
   // If bottommost_compression is set and we are compacting to the
   // bottommost level then we should use it.
   bool bottom_level = (level >= (vstorage->num_non_empty_levels() - 1));
-  if (moptions.bottommost_compressor != nullptr && bottom_level) {
-    return moptions.bottommost_compressor;
+  if (moptions.derived_bottommost_compressor != nullptr && bottom_level) {
+    return moptions.derived_bottommost_compressor;
   }
   // If the user has specified a different compression level for each level,
   // then pick the compression for that level.
-  if (!moptions.compressor_per_level.empty()) {
+  if (!moptions.derived_compressor_per_level.empty()) {
     // It is possible for level_ to be -1; in that case, we use level
     // 0's compression.  This occurs mostly in backwards compatibility
     // situations when the builder doesn't know what level the file
@@ -101,7 +101,8 @@ std::shared_ptr<Compressor> GetCompressor(const VersionStorageInfo* vstorage,
     assert(level == 0 || level >= base_level);
     int lvl = std::max(0, level - base_level + 1);
     int idx = std::min(
-        static_cast<int>(moptions.compressor_per_level.size()) - 1, lvl);
+        static_cast<int>(moptions.derived_compressor_per_level.size()) - 1,
+        lvl);
     // If not specified directly by the user, compressors in
     // compressor_per_level are instantiated using compression_opts. If the user
     // enabled bottommost_compression_opts, we need to create a new compressor
@@ -112,10 +113,10 @@ std::shared_ptr<Compressor> GetCompressor(const VersionStorageInfo* vstorage,
           moptions.compression_per_level[idx],
           moptions.bottommost_compression_opts);
     } else {
-      return moptions.compressor_per_level[idx];
+      return moptions.derived_compressor_per_level[idx];
     }
   } else {
-    return moptions.compressor;
+    return moptions.derived_compressor;
   }
 }
 

db/compaction/compaction_picker_fifo.cc

@@ -184,7 +184,7 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
             vstorage, ioptions_, mutable_cf_options, mutable_db_options,
             {comp_inputs}, 0, 16 * 1024 * 1024 /* output file size limit */,
             0 /* max compaction bytes, not applicable */,
-            0 /* output path ID */, mutable_cf_options.compressor,
+            0 /* output path ID */, mutable_cf_options.derived_compressor,
             Temperature::kUnknown, 0 /* max_subcompactions */, {},
             /* is manual */ false, /* trim_ts */ "",
             vstorage->CompactionScore(0),
@@ -419,7 +419,7 @@ Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction(
       vstorage, ioptions_, mutable_cf_options, mutable_db_options,
       std::move(inputs), 0, 0 /* output file size limit */,
       0 /* max compaction bytes, not applicable */, 0 /* output path ID */,
-      mutable_cf_options.compressor, compaction_target_temp,
+      mutable_cf_options.derived_compressor, compaction_target_temp,
       /* max_subcompactions */ 0, {}, /* is manual */ false, /* trim_ts */ "",
       vstorage->CompactionScore(0),
       /* is deletion compaction */ false, /* l0_files_might_overlap */ true,

db/db_basic_test.cc

@@ -12,6 +12,7 @@
 #include "db/db_test_util.h"
 #include "options/options_helper.h"
 #include "port/stack_trace.h"
+#include "rocksdb/compressor.h"
 #include "rocksdb/filter_policy.h"
 #include "rocksdb/flush_block_policy.h"
 #include "rocksdb/merge_operator.h"
@@ -21,7 +22,6 @@
 #include "table/block_based/block_based_table_reader.h"
 #include "table/block_based/block_builder.h"
 #include "test_util/sync_point.h"
-#include "util/compressor.h"
 #include "util/file_checksum_helper.h"
 #include "util/random.h"
 #include "utilities/counted_fs.h"