From 9f66b68652a33d550f6d57376c2ab4fbae700302 Mon Sep 17 00:00:00 2001 From: Matthias Petri Date: Thu, 26 Mar 2015 13:41:03 +1100 Subject: [PATCH 01/11] added read_only_mapper typedef --- include/sdsl/int_vector_mapper.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/sdsl/int_vector_mapper.hpp b/include/sdsl/int_vector_mapper.hpp index 7bdba2d01..ac49fd361 100644 --- a/include/sdsl/int_vector_mapper.hpp +++ b/include/sdsl/int_vector_mapper.hpp @@ -343,6 +343,10 @@ class temp_file_buffer template using bit_vector_mapper = int_vector_mapper<1,t_mode>; +template +using read_only_mapper = int_vector_mapper; + + } // end of namespace #endif From 0f4ff9253198582bb378f169bf2181cf28b72134 Mon Sep 17 00:00:00 2001 From: Matthias Petri Date: Thu, 26 Mar 2015 13:47:45 +1100 Subject: [PATCH 02/11] added write out buffer --- include/sdsl/int_vector_mapper.hpp | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/include/sdsl/int_vector_mapper.hpp b/include/sdsl/int_vector_mapper.hpp index ac49fd361..9cf637605 100644 --- a/include/sdsl/int_vector_mapper.hpp +++ b/include/sdsl/int_vector_mapper.hpp @@ -340,13 +340,31 @@ class temp_file_buffer } }; +// creates emtpy int_vector<> that will not be deleted +template +class write_out_buffer +{ + public: + static int_vector_mapper create(const std::string& key,const cache_config& config) { + auto file_name = cache_file_name(key,config); + auto tmp = create(file_name); + register_cache_file(key,config); + return std::move(tmp); + } + static int_vector_mapper create(const std::string& file_name) { + //write empty int_vector to init the file + int_vector tmp_vector; + store_to_file(tmp_vector,file_name); + return int_vector_mapper(file_name,false,false); + } +}; + template using bit_vector_mapper = int_vector_mapper<1,t_mode>; template using read_only_mapper = int_vector_mapper; - } // end of namespace #endif From d520061845752872c119be91d9c045781c80bcd0 Mon Sep 17 00:00:00 2001 From: Matthias Petri Date: Thu, 26 Mar 2015 13:51:12 +1100 Subject: [PATCH 03/11] simplified types --- include/sdsl/int_vector_mapper.hpp | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/include/sdsl/int_vector_mapper.hpp b/include/sdsl/int_vector_mapper.hpp index 9cf637605..a4304f47d 100644 --- a/include/sdsl/int_vector_mapper.hpp +++ b/include/sdsl/int_vector_mapper.hpp @@ -20,6 +20,10 @@ class int_vector_mapper typedef typename int_vector::value_type value_type; typedef typename int_vector::size_type size_type; typedef typename int_vector::int_width_type width_type; + typedef typename int_vector::const_iterator const_iterator; + typedef typename int_vector::iterator iterator; + typedef typename int_vector::const_reference const_reference; + typedef typename int_vector::reference reference; public: const size_type append_block_size = 1000000; private: @@ -219,39 +223,36 @@ class int_vector_mapper m_wrapper.m_size = bit_size; } - void resize(const size_type size) { static_assert(t_mode & std::ios_base::out,"int_vector_mapper: must be opened in in+out mode for 'resize'"); size_type size_in_bits = size * width(); bit_resize(size_in_bits); } - auto begin() -> typename int_vector::iterator { + iterator begin() { static_assert(t_mode & std::ios_base::out,"int_vector_mapper: must be opened in in+out mode for 'begin'"); return m_wrapper.begin(); } - auto end() -> typename int_vector::iterator { + iterator end() { static_assert(t_mode & std::ios_base::out,"int_vector_mapper: must be opened in in+out mode for 'end'"); return m_wrapper.end(); } - auto begin() const -> typename int_vector::const_iterator { + const_iterator begin() const { return m_wrapper.begin(); } - auto end() const -> typename int_vector::const_iterator { + const_iterator end() const { return m_wrapper.end(); } - auto cbegin() const -> typename int_vector::const_iterator { + const_iterator cbegin() const { return m_wrapper.begin(); } - auto cend() const -> typename int_vector::const_iterator { + const_iterator cend() const { return m_wrapper.end(); } - auto operator[](const size_type& idx) const - -> typename int_vector::const_reference { + const_reference operator[](const size_type& idx) const { return m_wrapper[idx]; } - auto operator[](const size_type& idx) - -> typename int_vector::reference { + reference operator[](const size_type& idx) { static_assert(t_mode & std::ios_base::out,"int_vector_mapper: must be opened in in+out mode for 'operator[]'"); return m_wrapper[idx]; } @@ -307,6 +308,10 @@ class int_vector_mapper bool empty() const { return m_wrapper.empty(); } + size_type max_size() const + { + return m_wrapper.max_size(); + } }; template From 3d7545f94036a30837e0a116f9f4525b4c6a5acf Mon Sep 17 00:00:00 2001 From: Matthias Petri Date: Thu, 26 Mar 2015 13:51:53 +1100 Subject: [PATCH 04/11] added const to read_only_mapper --- include/sdsl/int_vector_mapper.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/sdsl/int_vector_mapper.hpp b/include/sdsl/int_vector_mapper.hpp index a4304f47d..951b99e01 100644 --- a/include/sdsl/int_vector_mapper.hpp +++ b/include/sdsl/int_vector_mapper.hpp @@ -368,7 +368,7 @@ template using bit_vector_mapper = int_vector_mapper<1,t_mode>; template -using read_only_mapper = int_vector_mapper; +using read_only_mapper = const int_vector_mapper; } // end of namespace From fd8c257c8ad7ac0a48e9338ebff8131fca84725e Mon Sep 17 00:00:00 2001 From: Matthias Petri Date: Thu, 26 Mar 2015 14:06:46 +1100 Subject: [PATCH 05/11] fixes and tests for new features --- include/sdsl/int_vector_mapper.hpp | 105 +++++++++++++++++++---------- test/IntVectorMapperTest.cpp | 47 ++++++++++++- 2 files changed, 116 insertions(+), 36 deletions(-) diff --git a/include/sdsl/int_vector_mapper.hpp b/include/sdsl/int_vector_mapper.hpp index 951b99e01..1ec4b4822 100644 --- a/include/sdsl/int_vector_mapper.hpp +++ b/include/sdsl/int_vector_mapper.hpp @@ -35,7 +35,8 @@ class int_vector_mapper std::string m_file_name; bool m_delete_on_close; private: - void mmap_file() { + void mmap_file() + { if (!(t_mode&std::ios_base::out)) { // read only m_mapped_data = (uint8_t*)mmap(NULL, m_file_size_bytes, @@ -65,7 +66,8 @@ class int_vector_mapper int_vector_mapper(const int_vector_mapper&) = delete; int_vector_mapper& operator=(const int_vector_mapper&) = delete; public: - ~int_vector_mapper() { + ~int_vector_mapper() + { if (m_mapped_data) { if (t_mode&std::ios_base::out) { // write was possible if (m_data_offset) { @@ -116,7 +118,8 @@ class int_vector_mapper m_wrapper.m_data = nullptr; m_wrapper.m_size = 0; } - int_vector_mapper(int_vector_mapper&& ivm) { + int_vector_mapper(int_vector_mapper&& ivm) + { m_wrapper.m_data = ivm.m_wrapper.m_data; m_wrapper.m_size = ivm.m_wrapper.m_size; m_wrapper.width(ivm.m_wrapper.width()); @@ -127,7 +130,8 @@ class int_vector_mapper ivm.m_mapped_data = nullptr; ivm.m_fd = -1; } - int_vector_mapper& operator=(int_vector_mapper&& ivm) { + int_vector_mapper& operator=(int_vector_mapper&& ivm) + { m_wrapper.m_data = ivm.m_wrapper.m_data; m_wrapper.m_size = ivm.m_wrapper.m_size; m_wrapper.width(ivm.m_wrapper.width()); @@ -144,7 +148,8 @@ class int_vector_mapper int_vector_mapper(const std::string filename, bool is_plain = false, bool delete_on_close = false) : - m_file_name(filename), m_delete_on_close(delete_on_close) { + m_file_name(filename), m_delete_on_close(delete_on_close) + { size_type size_in_bits = 0; uint8_t int_width = t_width; { @@ -193,14 +198,17 @@ class int_vector_mapper } std::string file_name() const { return m_file_name; } width_type width() const { return m_wrapper.width(); } - void width(const uint8_t new_int_width) { + void width(const uint8_t new_int_width) + { static_assert(t_mode & std::ios_base::out,"int_vector_mapper: must be opened in in+out mode for 'width'"); m_wrapper.width(new_int_width); } - size_type size() const { + size_type size() const + { return m_wrapper.size(); } - void bit_resize(const size_type bit_size) { + void bit_resize(const size_type bit_size) + { static_assert(t_mode & std::ios_base::out,"int_vector_mapper: must be opened in in+out mode for 'bit_resize'"); size_type new_size_in_bytes = ((bit_size + 63) >> 6) << 3; if (m_file_size_bytes != new_size_in_bytes + m_data_offset) { @@ -223,52 +231,65 @@ class int_vector_mapper m_wrapper.m_size = bit_size; } - void resize(const size_type size) { + void resize(const size_type size) + { static_assert(t_mode & std::ios_base::out,"int_vector_mapper: must be opened in in+out mode for 'resize'"); size_type size_in_bits = size * width(); bit_resize(size_in_bits); } - iterator begin() { + iterator begin() + { static_assert(t_mode & std::ios_base::out,"int_vector_mapper: must be opened in in+out mode for 'begin'"); return m_wrapper.begin(); } - iterator end() { + iterator end() + { static_assert(t_mode & std::ios_base::out,"int_vector_mapper: must be opened in in+out mode for 'end'"); return m_wrapper.end(); } - const_iterator begin() const { + const_iterator begin() const + { return m_wrapper.begin(); } - const_iterator end() const { + const_iterator end() const + { return m_wrapper.end(); } - const_iterator cbegin() const { + const_iterator cbegin() const + { return m_wrapper.begin(); } - const_iterator cend() const { + const_iterator cend() const + { return m_wrapper.end(); } - const_reference operator[](const size_type& idx) const { + const_reference operator[](const size_type& idx) const + { return m_wrapper[idx]; } - reference operator[](const size_type& idx) { + reference operator[](const size_type& idx) + { static_assert(t_mode & std::ios_base::out,"int_vector_mapper: must be opened in in+out mode for 'operator[]'"); return m_wrapper[idx]; } const uint64_t* data() const { return m_wrapper.data(); } - uint64_t* data() { + uint64_t* data() + { static_assert(t_mode & std::ios_base::out,"int_vector_mapper: must be opened in in+out mode for 'data'"); return m_wrapper.data(); } - value_type get_int(size_type idx, const uint8_t len = 64) const { + value_type get_int(size_type idx, const uint8_t len = 64) const + { return m_wrapper.get_int(idx, len); } - void set_int(size_type idx, value_type x, const uint8_t len = 64) { + void set_int(size_type idx, value_type x, const uint8_t len = 64) + { static_assert(t_mode & std::ios_base::out,"int_vector_mapper: must be opened in in+out mode for 'set_int'"); m_wrapper.set_int(idx, x, len); } - void push_back(value_type x) { + void push_back(value_type x) + { static_assert(t_mode & std::ios_base::out,"int_vector_mapper: must be opened in in+out mode for 'push_back'"); if (capacity() < size() + 1) { size_type old_size = m_wrapper.m_size; @@ -280,32 +301,40 @@ class int_vector_mapper m_wrapper.m_size += width(); m_wrapper[size()-1] = x; } - size_type capacity() const { + size_type capacity() const + { size_t data_size_in_bits = 8 * (m_file_size_bytes - m_data_offset); return data_size_in_bits / width(); } - size_type bit_size() const { + size_type bit_size() const + { return m_wrapper.bit_size(); } template - bool operator==(const container& v) const { + bool operator==(const container& v) const + { return std::equal(begin(), end(), v.begin()); } - bool operator==(const int_vector& v) const { + bool operator==(const int_vector& v) const + { return m_wrapper == v; } - bool operator==(const int_vector_mapper& v) const { + bool operator==(const int_vector_mapper& v) const + { return m_wrapper == v.m_wrapper; } template - bool operator!=(const container& v) const { + bool operator!=(const container& v) const + { return !(*this==v); } - void flip() { + void flip() + { static_assert(t_mode & std::ios_base::out,"int_vector_mapper: must be opened in in+out mode for 'flip'"); m_wrapper.flip(); } - bool empty() const { + bool empty() const + { return m_wrapper.empty(); } size_type max_size() const @@ -318,7 +347,8 @@ template class temp_file_buffer { private: - static std::string tmp_file(const std::string& dir) { + static std::string tmp_file(const std::string& dir) + { char tmp_file_name[1024] = {0}; sprintf(tmp_file_name, "%s/tmp_mapper_file_XXXXXX.sdsl",dir.c_str()); int fd = mkstemps(tmp_file_name,5); @@ -329,15 +359,18 @@ class temp_file_buffer return std::string(tmp_file_name,strlen(tmp_file_name)); } public: - static int_vector_mapper create() { + static int_vector_mapper create() + { auto file_name = tmp_file("/tmp"); return create(file_name); } - static int_vector_mapper create(const cache_config& config) { + static int_vector_mapper create(const cache_config& config) + { auto file_name = tmp_file(config.dir); return create(file_name); } - static int_vector_mapper create(const std::string& file_name) { + static int_vector_mapper create(const std::string& file_name) + { //write empty int_vector to init the file int_vector tmp_vector; store_to_file(tmp_vector,file_name); @@ -350,13 +383,15 @@ template class write_out_buffer { public: - static int_vector_mapper create(const std::string& key,const cache_config& config) { + static int_vector_mapper create(const std::string& key,cache_config& config) + { auto file_name = cache_file_name(key,config); auto tmp = create(file_name); register_cache_file(key,config); return std::move(tmp); } - static int_vector_mapper create(const std::string& file_name) { + static int_vector_mapper create(const std::string& file_name) + { //write empty int_vector to init the file int_vector tmp_vector; store_to_file(tmp_vector,file_name); diff --git a/test/IntVectorMapperTest.cpp b/test/IntVectorMapperTest.cpp index d50b1181a..a927bb2b8 100644 --- a/test/IntVectorMapperTest.cpp +++ b/test/IntVectorMapperTest.cpp @@ -21,7 +21,8 @@ class IntVectorMapperTest : public ::testing::Test virtual ~IntVectorMapperTest() {} - virtual void SetUp() { + virtual void SetUp() + { std::mt19937_64 rng; { std::uniform_int_distribution distribution(1, 100000); @@ -279,6 +280,50 @@ TEST_F(IntVectorMapperTest, temp_buffer_test) } } +TEST_F(IntVectorMapperTest, read_only_mapper) +{ + for (const auto& size : vec_sizes) { + sdsl::int_vector<> vec(size); + sdsl::util::set_to_id(vec); + sdsl::store_to_file(vec,"tmp/bit_vector_mapper_test"); + { + sdsl::read_only_mapper<> rvec("tmp/bit_vector_mapper_test"); + ASSERT_EQ(rvec.width(),(uint8_t)64); + ASSERT_EQ(rvec.size(),(size_t)vec.size()); + ASSERT_TRUE(std::equal(rvec.begin(),rvec.end(),vec.begin())); + } + // check that the file is still there + std::ifstream cfs("tmp/bit_vector_mapper_test"); + ASSERT_TRUE(cfs.is_open()); + sdsl::remove("tmp/bit_vector_mapper_test"); + } +} + +TEST_F(IntVectorMapperTest, write_out_buffer) +{ + for (const auto& size : vec_sizes) { + sdsl::int_vector<> vec(size); + sdsl::util::set_to_id(vec); + std::string tmp_file_name = "tmp/write_out_buffer.sdsl"; + { + auto buf = sdsl::write_out_buffer<31>::create(tmp_file_name); + ASSERT_EQ(buf.file_name(),tmp_file_name); + ASSERT_EQ(buf.width(),(uint8_t)31); + ASSERT_EQ(buf.size(),(size_t)0); + ASSERT_TRUE(buf.empty()); + for (const auto& val : vec) { + buf.push_back(val); + } + ASSERT_EQ(buf.size(),vec.size()); + ASSERT_TRUE(std::equal(buf.begin(),buf.end(),vec.begin())); + } + // check that the file is NOT gone + std::ifstream cfs(tmp_file_name); + ASSERT_TRUE(cfs.is_open()); + sdsl::remove(tmp_file_name); + } +} + } // namespace int main(int argc, char** argv) From 76fdbd4fb8aac19e4d594da7302ac9552ad62f78 Mon Sep 17 00:00:00 2001 From: Matthias Petri Date: Fri, 10 Apr 2015 19:43:10 +1000 Subject: [PATCH 06/11] added whitespace --- include/sdsl/int_vector_mapper.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/sdsl/int_vector_mapper.hpp b/include/sdsl/int_vector_mapper.hpp index 1ec4b4822..b81bb9864 100644 --- a/include/sdsl/int_vector_mapper.hpp +++ b/include/sdsl/int_vector_mapper.hpp @@ -343,6 +343,7 @@ class int_vector_mapper } }; + template class temp_file_buffer { From 24ac2d6ca4b0b8b211d91302556e640bc1c5622f Mon Sep 17 00:00:00 2001 From: Matthias Petri Date: Fri, 10 Apr 2015 19:50:35 +1000 Subject: [PATCH 07/11] removed whitespace and added comment --- include/sdsl/int_vector_mapper.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/sdsl/int_vector_mapper.hpp b/include/sdsl/int_vector_mapper.hpp index b81bb9864..016c2022f 100644 --- a/include/sdsl/int_vector_mapper.hpp +++ b/include/sdsl/int_vector_mapper.hpp @@ -343,7 +343,7 @@ class int_vector_mapper } }; - +/* temporary buffer that gets deleted later */ template class temp_file_buffer { From b2e110a089c91a4c2cd46ff9890680f2c89a520c Mon Sep 17 00:00:00 2001 From: Matthias Petri Date: Fri, 10 Apr 2015 19:53:12 +1000 Subject: [PATCH 08/11] fixed more comments --- include/sdsl/int_vector_mapper.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/sdsl/int_vector_mapper.hpp b/include/sdsl/int_vector_mapper.hpp index 016c2022f..765ac28ff 100644 --- a/include/sdsl/int_vector_mapper.hpp +++ b/include/sdsl/int_vector_mapper.hpp @@ -393,7 +393,7 @@ class write_out_buffer } static int_vector_mapper create(const std::string& file_name) { - //write empty int_vector to init the file + // write empty int_vector to init the file int_vector tmp_vector; store_to_file(tmp_vector,file_name); return int_vector_mapper(file_name,false,false); From 5a6a8bc323346d0b7e2c0e7d763ec2b1b1b7002d Mon Sep 17 00:00:00 2001 From: Matthias Petri Date: Fri, 10 Apr 2015 20:11:40 +1000 Subject: [PATCH 09/11] more comments --- include/sdsl/int_vector_mapper.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/sdsl/int_vector_mapper.hpp b/include/sdsl/int_vector_mapper.hpp index 765ac28ff..3452ced17 100644 --- a/include/sdsl/int_vector_mapper.hpp +++ b/include/sdsl/int_vector_mapper.hpp @@ -88,6 +88,7 @@ class int_vector_mapper } } + /* unmap data */ munmap(m_mapped_data, m_file_size_bytes); if (t_mode&std::ios_base::out) { From 292cf82f1e06717f4f9b97db3bce8d2a97ac8790 Mon Sep 17 00:00:00 2001 From: Matthias Petri Date: Fri, 10 Apr 2015 20:24:14 +1000 Subject: [PATCH 10/11] comment on deleted constructors --- include/sdsl/int_vector_mapper.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/sdsl/int_vector_mapper.hpp b/include/sdsl/int_vector_mapper.hpp index 3452ced17..eae788a92 100644 --- a/include/sdsl/int_vector_mapper.hpp +++ b/include/sdsl/int_vector_mapper.hpp @@ -62,6 +62,7 @@ class int_vector_mapper } } public: + /* maps a resource. thus should not be copied or default constructed */ int_vector_mapper() = delete; int_vector_mapper(const int_vector_mapper&) = delete; int_vector_mapper& operator=(const int_vector_mapper&) = delete; From 2bed5f5c364c096421b5b57f5753e7f533b746ad Mon Sep 17 00:00:00 2001 From: Matthias Petri Date: Fri, 10 Apr 2015 20:47:41 +1000 Subject: [PATCH 11/11] comment on deleted constructors --- include/sdsl/int_vector_mapper.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/sdsl/int_vector_mapper.hpp b/include/sdsl/int_vector_mapper.hpp index eae788a92..cfa4764a3 100644 --- a/include/sdsl/int_vector_mapper.hpp +++ b/include/sdsl/int_vector_mapper.hpp @@ -408,6 +408,6 @@ using bit_vector_mapper = int_vector_mapper<1,t_mode>; template using read_only_mapper = const int_vector_mapper; -} // end of namespace +} #endif