Merge "Implement FuseBlockDataProvider" am: bfd6cf59dc
am: 15a899363e
Change-Id: I58a37b32ce5217c968471c8095d357ca5f4061b7
diff --git a/fuse_sideload/Android.bp b/fuse_sideload/Android.bp
index 8548548..9bf19eb 100644
--- a/fuse_sideload/Android.bp
+++ b/fuse_sideload/Android.bp
@@ -34,6 +34,10 @@
"include",
],
+ static_libs: [
+ "libotautil",
+ ],
+
shared_libs: [
"libbase",
"libcrypto",
diff --git a/fuse_sideload/fuse_provider.cpp b/fuse_sideload/fuse_provider.cpp
index 58786f5..5ee6e24 100644
--- a/fuse_sideload/fuse_provider.cpp
+++ b/fuse_sideload/fuse_provider.cpp
@@ -27,8 +27,11 @@
#include <functional>
#include <android-base/file.h>
+#include <android-base/logging.h>
+#include <android-base/strings.h>
#include "fuse_sideload.h"
+#include "otautil/sysutil.h"
FuseFileDataProvider::FuseFileDataProvider(const std::string& path, uint32_t block_size) {
struct stat sb;
@@ -69,3 +72,79 @@
void FuseFileDataProvider::Close() {
fd_.reset();
}
+
+FuseBlockDataProvider::FuseBlockDataProvider(uint64_t file_size, uint32_t fuse_block_size,
+ android::base::unique_fd&& fd,
+ uint32_t source_block_size, RangeSet ranges)
+ : FuseDataProvider(file_size, fuse_block_size),
+ fd_(std::move(fd)),
+ source_block_size_(source_block_size),
+ ranges_(std::move(ranges)) {
+ // Make sure the offset is also aligned with the blocks on the block device when we call
+ // ReadBlockAlignedData().
+ CHECK_EQ(0, fuse_block_size_ % source_block_size_);
+}
+
+bool FuseBlockDataProvider::ReadBlockAlignedData(uint8_t* buffer, uint32_t fetch_size,
+ uint32_t start_block) const {
+ uint64_t offset = static_cast<uint64_t>(start_block) * fuse_block_size_;
+ if (fetch_size > file_size_ || offset > file_size_ - fetch_size) {
+ LOG(ERROR) << "Out of bound read, offset: " << offset << ", fetch size: " << fetch_size
+ << ", file size " << file_size_;
+ return false;
+ }
+
+ auto read_ranges =
+ ranges_.GetSubRanges(offset / source_block_size_, fetch_size / source_block_size_);
+ if (!read_ranges) {
+ return false;
+ }
+
+ uint8_t* next_out = buffer;
+ for (const auto& [range_start, range_end] : read_ranges.value()) {
+ uint64_t bytes_start = static_cast<uint64_t>(range_start) * source_block_size_;
+ uint64_t bytes_to_read = static_cast<uint64_t>(range_end - range_start) * source_block_size_;
+ if (!android::base::ReadFullyAtOffset(fd_, next_out, bytes_to_read, bytes_start)) {
+ PLOG(ERROR) << "Failed to read " << bytes_to_read << " bytes at offset " << bytes_start;
+ return false;
+ }
+
+ next_out += bytes_to_read;
+ }
+
+ if (uint64_t tailing_bytes = fetch_size % source_block_size_; tailing_bytes != 0) {
+ // Calculate the offset to last partial block.
+ uint64_t tailing_offset =
+ read_ranges.value()
+ ? static_cast<uint64_t>((read_ranges->cend() - 1)->second) * source_block_size_
+ : static_cast<uint64_t>(start_block) * source_block_size_;
+ if (!android::base::ReadFullyAtOffset(fd_, next_out, tailing_bytes, tailing_offset)) {
+ PLOG(ERROR) << "Failed to read tailing " << tailing_bytes << " bytes at offset "
+ << tailing_offset;
+ return false;
+ }
+ }
+ return true;
+}
+
+std::unique_ptr<FuseBlockDataProvider> FuseBlockDataProvider::CreateFromBlockMap(
+ const std::string& block_map_path, uint32_t fuse_block_size) {
+ auto block_map = BlockMapData::ParseBlockMapFile(block_map_path);
+ if (!block_map) {
+ return nullptr;
+ }
+
+ android::base::unique_fd fd(TEMP_FAILURE_RETRY(open(block_map.path().c_str(), O_RDONLY)));
+ if (fd == -1) {
+ PLOG(ERROR) << "Failed to open " << block_map.path();
+ return nullptr;
+ }
+
+ return std::unique_ptr<FuseBlockDataProvider>(
+ new FuseBlockDataProvider(block_map.file_size(), fuse_block_size, std::move(fd),
+ block_map.block_size(), block_map.block_ranges()));
+}
+
+void FuseBlockDataProvider::Close() {
+ fd_.reset();
+}
diff --git a/fuse_sideload/include/fuse_provider.h b/fuse_sideload/include/fuse_provider.h
index 59059cf..8d4ea40 100644
--- a/fuse_sideload/include/fuse_provider.h
+++ b/fuse_sideload/include/fuse_provider.h
@@ -18,10 +18,13 @@
#include <stdint.h>
+#include <memory>
#include <string>
#include <android-base/unique_fd.h>
+#include "otautil/rangeset.h"
+
// This is the base class to read data from source and provide the data to FUSE.
class FuseDataProvider {
public:
@@ -70,3 +73,28 @@
// The underlying source to read data from.
android::base::unique_fd fd_;
};
+
+// This class parses a block map and reads data from the underlying block device.
+class FuseBlockDataProvider : public FuseDataProvider {
+ public:
+ // Constructs the fuse provider from the block map.
+ static std::unique_ptr<FuseBlockDataProvider> CreateFromBlockMap(
+ const std::string& block_map_path, uint32_t fuse_block_size);
+
+ RangeSet ranges() const {
+ return ranges_;
+ }
+ bool ReadBlockAlignedData(uint8_t* buffer, uint32_t fetch_size,
+ uint32_t start_block) const override;
+ void Close() override;
+
+ private:
+ FuseBlockDataProvider(uint64_t file_size, uint32_t fuse_block_size, android::base::unique_fd&& fd,
+ uint32_t source_block_size, RangeSet ranges);
+ // The underlying block device to read data from.
+ android::base::unique_fd fd_;
+ // The block size of the source block device.
+ uint32_t source_block_size_;
+ // The block ranges from the source block device that consist of the file
+ RangeSet ranges_;
+};
diff --git a/minadbd/Android.bp b/minadbd/Android.bp
index 007e505..afd57ad 100644
--- a/minadbd/Android.bp
+++ b/minadbd/Android.bp
@@ -43,6 +43,10 @@
"minadbd_services.cpp",
],
+ static_libs: [
+ "libotautil",
+ ],
+
shared_libs: [
"libadbd",
"libbase",
@@ -96,6 +100,7 @@
static_libs: [
"libminadbd_services",
"libfusesideload",
+ "libotautil",
"libadbd",
"libcrypto",
],
diff --git a/otautil/include/otautil/rangeset.h b/otautil/include/otautil/rangeset.h
index e91d02c..a18c30e 100644
--- a/otautil/include/otautil/rangeset.h
+++ b/otautil/include/otautil/rangeset.h
@@ -18,6 +18,7 @@
#include <stddef.h>
+#include <optional>
#include <string>
#include <utility>
#include <vector>
@@ -49,6 +50,12 @@
// bounds. For example, "3,5" contains blocks 3 and 4. So "3,5" and "5,7" are not overlapped.
bool Overlaps(const RangeSet& other) const;
+ // Returns a subset of ranges starting from |start_index| with respect to the original range. The
+ // output range will have |num_of_blocks| blocks in size. Returns std::nullopt if the input is
+ // invalid. e.g. RangeSet({{0, 5}, {10, 15}}).GetSubRanges(1, 5) returns
+ // RangeSet({{1, 5}, {10, 11}}).
+ std::optional<RangeSet> GetSubRanges(size_t start_index, size_t num_of_blocks) const;
+
// Returns a vector of RangeSets that contain the same set of blocks represented by the current
// RangeSet. The RangeSets in the vector contain similar number of blocks, with a maximum delta
// of 1-block between any two of them. For example, 14 blocks would be split into 4 + 4 + 3 + 3,
diff --git a/otautil/rangeset.cpp b/otautil/rangeset.cpp
index 5ab8e08..8ee99dd 100644
--- a/otautil/rangeset.cpp
+++ b/otautil/rangeset.cpp
@@ -184,6 +184,58 @@
return false;
}
+std::optional<RangeSet> RangeSet::GetSubRanges(size_t start_index, size_t num_of_blocks) const {
+ size_t end_index = start_index + num_of_blocks; // The index of final block to read plus one
+ if (start_index > end_index || end_index > blocks_) {
+ LOG(ERROR) << "Failed to get the sub ranges for start_index " << start_index
+ << " num_of_blocks " << num_of_blocks
+ << " total number of blocks the range contains is " << blocks_;
+ return std::nullopt;
+ }
+
+ if (num_of_blocks == 0) {
+ LOG(WARNING) << "num_of_blocks is zero when calling GetSubRanges()";
+ return RangeSet();
+ }
+
+ RangeSet result;
+ size_t current_index = 0;
+ for (const auto& [range_start, range_end] : ranges_) {
+ CHECK_LT(range_start, range_end);
+ size_t blocks_in_range = range_end - range_start;
+ // Linear search to skip the ranges until we reach start_block.
+ if (current_index + blocks_in_range <= start_index) {
+ current_index += blocks_in_range;
+ continue;
+ }
+
+ size_t trimmed_range_start = range_start;
+ // We have found the first block range to read, trim the heading blocks.
+ if (current_index < start_index) {
+ trimmed_range_start += start_index - current_index;
+ }
+ // Trim the trailing blocks if the last range has more blocks than desired; also return the
+ // result.
+ if (current_index + blocks_in_range >= end_index) {
+ size_t trimmed_range_end = range_end - (current_index + blocks_in_range - end_index);
+ if (!result.PushBack({ trimmed_range_start, trimmed_range_end })) {
+ return std::nullopt;
+ }
+
+ return result;
+ }
+
+ if (!result.PushBack({ trimmed_range_start, range_end })) {
+ return std::nullopt;
+ }
+ current_index += blocks_in_range;
+ }
+
+ LOG(ERROR) << "Failed to construct byte ranges to read, start_block: " << start_index
+ << ", num_of_blocks: " << num_of_blocks << " total number of blocks: " << blocks_;
+ return std::nullopt;
+}
+
// Ranges in the the set should be mutually exclusive; and they're sorted by the start block.
SortedRangeSet::SortedRangeSet(std::vector<Range>&& pairs) : RangeSet(std::move(pairs)) {
std::sort(ranges_.begin(), ranges_.end());
diff --git a/otautil/sysutil.cpp b/otautil/sysutil.cpp
index 2b48618..420db4c 100644
--- a/otautil/sysutil.cpp
+++ b/otautil/sysutil.cpp
@@ -94,6 +94,11 @@
remaining_blocks -= range_blocks;
}
+ if (remaining_blocks != 0) {
+ LOG(ERROR) << "Invalid ranges: remaining blocks " << remaining_blocks;
+ return {};
+ }
+
return BlockMapData(block_dev, file_size, blksize, std::move(ranges));
}
diff --git a/tests/Android.bp b/tests/Android.bp
index ec2124a..67a65ae 100644
--- a/tests/Android.bp
+++ b/tests/Android.bp
@@ -118,6 +118,7 @@
static_libs: libapplypatch_static_libs + librecovery_static_libs + [
"librecovery_ui",
+ "libfusesideload",
"libminui",
"libotautil",
"libupdater",
diff --git a/tests/unit/fuse_provider_test.cpp b/tests/unit/fuse_provider_test.cpp
new file mode 100644
index 0000000..c5995dd
--- /dev/null
+++ b/tests/unit/fuse_provider_test.cpp
@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdint.h>
+#include <unistd.h>
+
+#include <functional>
+#include <string>
+#include <vector>
+
+#include <android-base/file.h>
+#include <android-base/logging.h>
+#include <android-base/strings.h>
+#include <android-base/unique_fd.h>
+#include <gtest/gtest.h>
+
+#include "fuse_provider.h"
+#include "fuse_sideload.h"
+#include "install/install.h"
+
+TEST(FuseBlockMapTest, CreateFromBlockMap_smoke) {
+ TemporaryFile fake_block_device;
+ std::vector<std::string> lines = {
+ fake_block_device.path, "10000 4096", "3", "10 11", "20 21", "22 23",
+ };
+
+ TemporaryFile temp_file;
+ android::base::WriteStringToFile(android::base::Join(lines, '\n'), temp_file.path);
+ auto block_map_data = FuseBlockDataProvider::CreateFromBlockMap(temp_file.path, 4096);
+
+ ASSERT_TRUE(block_map_data);
+ ASSERT_EQ(10000, block_map_data->file_size());
+ ASSERT_EQ(4096, block_map_data->fuse_block_size());
+ ASSERT_EQ(RangeSet({ { 10, 11 }, { 20, 21 }, { 22, 23 } }), block_map_data->ranges());
+}
+
+TEST(FuseBlockMapTest, ReadBlockAlignedData_smoke) {
+ std::string content;
+ content.reserve(40960);
+ for (char c = 0; c < 10; c++) {
+ content += std::string(4096, c);
+ }
+ TemporaryFile fake_block_device;
+ ASSERT_TRUE(android::base::WriteStringToFile(content, fake_block_device.path));
+
+ std::vector<std::string> lines = {
+ fake_block_device.path,
+ "20000 4096",
+ "1",
+ "0 5",
+ };
+ TemporaryFile temp_file;
+ android::base::WriteStringToFile(android::base::Join(lines, '\n'), temp_file.path);
+ auto block_map_data = FuseBlockDataProvider::CreateFromBlockMap(temp_file.path, 4096);
+
+ std::vector<uint8_t> result(2000);
+ ASSERT_TRUE(block_map_data->ReadBlockAlignedData(result.data(), 2000, 1));
+ ASSERT_EQ(std::vector<uint8_t>(content.begin() + 4096, content.begin() + 6096), result);
+
+ result.resize(20000);
+ ASSERT_TRUE(block_map_data->ReadBlockAlignedData(result.data(), 20000, 0));
+ ASSERT_EQ(std::vector<uint8_t>(content.begin(), content.begin() + 20000), result);
+}
+
+TEST(FuseBlockMapTest, ReadBlockAlignedData_large_fuse_block) {
+ std::string content;
+ for (char c = 0; c < 10; c++) {
+ content += std::string(4096, c);
+ }
+
+ TemporaryFile temp_file;
+ ASSERT_TRUE(android::base::WriteStringToFile(content, temp_file.path));
+
+ std::vector<std::string> lines = {
+ temp_file.path, "36384 4096", "2", "0 5", "6 10",
+ };
+ TemporaryFile block_map;
+ ASSERT_TRUE(android::base::WriteStringToFile(android::base::Join(lines, '\n'), block_map.path));
+
+ auto block_map_data = FuseBlockDataProvider::CreateFromBlockMap(block_map.path, 16384);
+ ASSERT_TRUE(block_map_data);
+
+ std::vector<uint8_t> result(20000);
+ // Out of bound read
+ ASSERT_FALSE(block_map_data->ReadBlockAlignedData(result.data(), 20000, 2));
+ ASSERT_TRUE(block_map_data->ReadBlockAlignedData(result.data(), 20000, 1));
+ // expected source block contains: 4, 6-9
+ std::string expected = content.substr(16384, 4096) + content.substr(24576, 15904);
+ ASSERT_EQ(std::vector<uint8_t>(expected.begin(), expected.end()), result);
+}
diff --git a/tests/unit/rangeset_test.cpp b/tests/unit/rangeset_test.cpp
index fc72f2f..699f933 100644
--- a/tests/unit/rangeset_test.cpp
+++ b/tests/unit/rangeset_test.cpp
@@ -18,6 +18,7 @@
#include <sys/types.h>
#include <limits>
+#include <optional>
#include <vector>
#include <gtest/gtest.h>
@@ -248,6 +249,29 @@
ASSERT_EQ("6,1,3,4,6,15,22", RangeSet::Parse("6,1,3,4,6,15,22").ToString());
}
+TEST(RangeSetTest, GetSubRanges_invalid) {
+ RangeSet range0({ { 1, 11 }, { 20, 30 } });
+ ASSERT_FALSE(range0.GetSubRanges(0, 21)); // too many blocks
+ ASSERT_FALSE(range0.GetSubRanges(21, 1)); // start block OOB
+}
+
+TEST(RangeSetTest, GetSubRanges_empty) {
+ RangeSet range0({ { 1, 11 }, { 20, 30 } });
+ ASSERT_EQ(RangeSet{}, range0.GetSubRanges(1, 0)); // empty num_of_blocks
+}
+
+TEST(RangeSetTest, GetSubRanges_smoke) {
+ RangeSet range0({ { 10, 11 } });
+ ASSERT_EQ(RangeSet({ { 10, 11 } }), range0.GetSubRanges(0, 1));
+
+ RangeSet range1({ { 10, 11 }, { 20, 21 }, { 30, 31 } });
+ ASSERT_EQ(range1, range1.GetSubRanges(0, 3));
+ ASSERT_EQ(RangeSet({ { 20, 21 } }), range1.GetSubRanges(1, 1));
+
+ RangeSet range2({ { 1, 11 }, { 20, 25 }, { 30, 35 } });
+ ASSERT_EQ(RangeSet({ { 10, 11 }, { 20, 25 }, { 30, 31 } }), range2.GetSubRanges(9, 7));
+}
+
TEST(SortedRangeSetTest, Insert) {
SortedRangeSet rs({ { 2, 3 }, { 4, 6 }, { 8, 14 } });
rs.Insert({ 1, 2 });
diff --git a/tests/unit/sysutil_test.cpp b/tests/unit/sysutil_test.cpp
index 3466e8e..64b8956 100644
--- a/tests/unit/sysutil_test.cpp
+++ b/tests/unit/sysutil_test.cpp
@@ -67,7 +67,7 @@
"/dev/abc",
"42949672950 4294967295",
"1",
- "0 9",
+ "0 10",
};
TemporaryFile temp_file;