Update to libbbf 1.1.0

ef1500 · ef1500 · commit e4caad9f69c7 · 2026-01-01T22:02:42.000-08:00
diff --git a/setup.py b/setup.py
@@ -24,7 +24,8 @@
         "libbbf",
         [
             "src/bindings.cpp",
-            "src/libbbf.cpp"
+            "src/libbbf.cpp",
+            "src/xxhash.c"
         ],
         include_dirs=["src"],
         cxx_std=17,
@@ -33,7 +34,7 @@
 
 setup(
     name="libbbf",
-    version="0.2.10",
+    version="0.2.12",
     author="EF1500",
     author_email="rosemilovelockofficial@proton.me",
     description="Bound Book Format (BBF) tools and bindings",
diff --git a/src/bbf_reader.h b/src/bbf_reader.h
@@ -4,7 +4,7 @@
 #include "xxhash.h"
 
 #include <string>
-#include <string_view> // C++17: Essential for zero-copy parsing
+#include <string_view> 
 #include <vector>
 #include <map>
 #include <cstring>
@@ -78,7 +78,6 @@ struct MemoryMappedFile {
 
 class BBFReader {
 private:
-    // Cached pointers to avoid recalculating offsets repeatedly
     const char* data_ptr = nullptr;
     const BBFSection* sections_ = nullptr;
     const BBFMetadata* meta_ = nullptr;
@@ -108,8 +107,9 @@ class BBFReader {
         std::memcpy(&footer, data_ptr + mmap.size - sizeof(BBFFooter), sizeof(BBFFooter));
         if (std::memcmp(footer.magic, "BBF1", 4) != 0) return;
 
-        // Cache Table Pointers
-        // Note: In production, you should add bounds checks here to ensure offsets are within mmap.size
+        // Safety: Ensure tables point inside the file
+        if (footer.assetTableOffset >= mmap.size || footer.pageTableOffset >= mmap.size) return;
+
         sections_ = reinterpret_cast<const BBFSection*>(data_ptr + footer.sectionTableOffset);
         meta_     = reinterpret_cast<const BBFMetadata*>(data_ptr + footer.metaTableOffset);
         pages_    = reinterpret_cast<const BBFPageEntry*>(data_ptr + footer.pageTableOffset);
@@ -121,12 +121,8 @@ class BBFReader {
         isValid = true;
     }
 
-    // Optimized: Returns string_view (no allocation)
-    // Helper to allow returning std::string for legacy binding support if needed, 
-    // but internal logic should prefer views.
     std::string_view getStringView(uint32_t offset) const {
         if (offset >= stringPoolSize_) return {};
-        // Requires strings in file to be null-terminated.
         return std::string_view(stringPool_ + offset);
     }
 
@@ -140,9 +136,8 @@ class BBFReader {
         std::vector<PySection> result;
         if (!isValid) return result;
         
-        result.reserve(footer.sectionCount); // Optimization: Reserve memory
+        result.reserve(footer.sectionCount);
         for (uint32_t i = 0; i < footer.sectionCount; i++) {
-            // Explicit conversion to std::string here is okay as we are handing off to Python
             result.push_back({
                 std::string(getStringView(sections_[i].sectionTitleOffset)), 
                 sections_[i].sectionStartIndex, 
@@ -166,21 +161,14 @@ class BBFReader {
         return result;
     }
 
-    // Zero-copy accessor for PyBind
-    // Returns {pointer, size}
     std::pair<const char*, size_t> getPageRaw(uint32_t pageIndex) const {
         if (!isValid || pageIndex >= footer.pageCount) return {nullptr, 0};
         
-        // Indirect addressing: Page -> Asset -> Offset/Length
         const auto& asset = assets_[pages_[pageIndex].assetIndex];
-        return { data_ptr + asset.offset, asset.length };
-    }
+        // Bounds check on asset
+        if (asset.offset + asset.length > mmap.size) return {nullptr, 0};
 
-    // Legacy support (copies data)
-    std::string getPageBytes(uint32_t pageIndex) const {
-        auto raw = getPageRaw(pageIndex);
-        if (!raw.first) return "";
-        return std::string(raw.first, raw.second);
+        return { data_ptr + asset.offset, static_cast<size_t>(asset.length) };
     }
 
     std::map<std::string, uint64_t> getPageInfo(uint32_t pageIndex) const {
@@ -191,56 +179,62 @@ class BBFReader {
             {"length", asset.length},
             {"offset", asset.offset},
             {"hash", asset.xxh3Hash},
-            {"type", asset.type}
+            {"type", asset.type},
+            {"decodedLength", asset.decodedLength} // ADDED: v1.1 Spec
         };
     }
 
-    bool verify() const {
-        if (!isValid) return false;
+    // Returns -1 for Success, -2 for Directory Error, or >=0 for Asset Index Error
+    int64_t verify() const {
+        if (!isValid) return -2;
         
         // 1. Directory Hash Check
         size_t metaStart = footer.stringPoolOffset;
         size_t metaSize = mmap.size - sizeof(BBFFooter) - metaStart;
-        if (XXH3_64bits(data_ptr + metaStart, metaSize) != footer.indexHash) return false;
+        if (XXH3_64bits(data_ptr + metaStart, metaSize) != footer.indexHash) return -2;
 
         // 2. Asset Integrity Check
         size_t count = footer.assetCount;
-        const auto* local_assets = assets_; // Copy pointer for lambda capture
+        const auto* local_assets = assets_;
         const auto* local_data = data_ptr;
+        size_t max_size = mmap.size;
 
-        auto verifyRange = [local_assets, local_data](size_t start, size_t end) -> bool {
+        // Lambda returns -1 if OK, or the index if Bad
+        auto verifyRange = [local_assets, local_data, max_size](size_t start, size_t end) -> int64_t {
             for (size_t i = start; i < end; ++i) {
                 const auto& a = local_assets[i];
+                // Bounds check before hash
+                if (a.offset + a.length > max_size) return (int64_t)i;
+                
                 if (XXH3_64bits((const uint8_t*)local_data + a.offset, a.length) != a.xxh3Hash) {
-                    return false;
+                    return (int64_t)i; // Return the corrupted index
                 }
             }
-            return true;
+            return -1; // Success
         };
 
-        // Optimization: Don't spawn threads for small files
         size_t numThreads = std::thread::hardware_concurrency();
         if (numThreads == 0) numThreads = 1;
 
-        // Heuristic: If assets < 128, threading overhead > hashing gain
         if (count < 128 || numThreads == 1) {
             return verifyRange(0, count);
         }
 
         size_t chunkSize = count / numThreads;
-        std::vector<std::future<bool>> futures;
+        std::vector<std::future<int64_t>> futures; // Changed from bool to int64_t
         futures.reserve(numThreads);
 
         for (size_t i = 0; i < numThreads; ++i) {
             size_t start = i * chunkSize;
             size_t end = (i == numThreads - 1) ? count : start + chunkSize;
-            // Launch async
             futures.push_back(std::async(std::launch::async, verifyRange, start, end));
         }
 
+        // Check results
         for (auto& f : futures) {
-            if (!f.get()) return false;
+            int64_t result = f.get();
+            if (result != -1) return result; // Bubble up the error index
         }
-        return true;
+        return -1; // All good
     }
 };
diff --git a/src/bbfenc.cpp b/src/bbfenc.cpp
@@ -1,3 +1,5 @@
+#define NOMINMAX
+
 #include "libbbf.h"
 #include "xxhash.h"
 #include <iostream>
@@ -133,13 +135,23 @@ class BBFReader
     {
         if (!mmap.map(path))
             return false;
+        
+        // Basic size check
         if (mmap.size < sizeof(BBFHeader) + sizeof(BBFFooter))
             return false;
 
+        // Read the fixed-size part of the header first
         std::memcpy(&header, mmap.data, sizeof(BBFHeader));
+        
         if (std::memcmp(header.magic, "BBF1", 4) != 0)
             return false;
 
+        // FUTURE PROOFING: 
+        // If header.headerLen > sizeof(BBFHeader), we know there is extra data 
+        // in the header we should ignore. We don't need to do anything right now 
+        // because we read assets via absolute offsets, but it's good to know.
+
+        // Read Footer
         std::memcpy(&footer, (uint8_t *)mmap.data + mmap.size - sizeof(BBFFooter), sizeof(BBFFooter));
         if (std::memcmp(footer.magic, "BBF1", 4) != 0)
             return false;
@@ -574,9 +586,13 @@ int main(int argc, char *argv[])
             for (uint32_t i = start; i < end; ++i)
             {
                 const auto &asset = assets[pages[i].assetIndex];
-                std::string outPath = (fs::path(outDir) / ("p" + std::to_string(i + 1) + ((asset.type == 1) ? ".avif" : ".png"))).string();
 
-                // Optimized: Direct write from mapped memory
+                // FIX: Use the library function to get the extension
+                // This automatically handles PNG, JPG, AVIF, JXL, etc.
+                std::string ext = MediaTypeToStr(asset.type);
+
+                std::string outPath = (fs::path(outDir) / ("p" + std::to_string(i + 1) + ext)).string();
+
                 std::ofstream ofs(outPath, std::ios::binary);
                 ofs.write((const char *)reader.mmap.data + asset.offset, asset.length);
             }
@@ -686,7 +702,8 @@ int main(int argc, char *argv[])
         for (uint32_t i = 0; i < manifest.size(); ++i)
         {
             std::string ext = fs::path(manifest[i].path).extension().string();
-            uint8_t type = (ext == ".avif" || ext == ".AVIF") ? 1 : 2;
+            BBFMediaType mediaType = detectTypeFromExtension(ext);
+            uint8_t type = static_cast<uint8_t>(mediaType);
             builder.addPage(manifest[i].path, type);
             fileToPage[manifest[i].filename] = i;
         }
diff --git a/src/bindings.cpp b/src/bindings.cpp
@@ -26,21 +26,20 @@ PYBIND11_MODULE(libbbf, m) {
     py::class_<BBFReader>(m, "BBFReader")
         .def(py::init<const std::string &>())
         .def_readonly("is_valid", &BBFReader::isValid)
-        .def_readonly("footer", &BBFReader::footer) // Optional: expose footer struct directly if bound
+        // We don't need to expose footer struct directly unless you wrote a binding for BBFFooter
         .def("get_page_count", [](BBFReader& r) { return r.footer.pageCount; })
         .def("get_asset_count", [](BBFReader& r) { return r.footer.assetCount; })
         
         .def("verify", &BBFReader::verify, 
-             py::call_guard<py::gil_scoped_release>(), // IMPORTANT: Release GIL during long hashing
-             "Verify integrity of index and assets. Multithreaded.")
+             py::call_guard<py::gil_scoped_release>(), 
+             "Verify integrity. Returns: -1 (Success), -2 (Directory Fail), or >=0 (Index of corrupt asset).")
 
         .def("get_sections", [](BBFReader& r) {
-            // Optimizing the conversion loop
             py::list result;
             const auto sections = r.getSections();
             for (const auto& s : sections) {
                 py::dict d;
-                d["title"] = s.title; // Moves string
+                d["title"] = s.title; 
                 d["startPage"] = s.startPage;
                 d["parent"] = s.parent;
                 result.append(d);
@@ -50,26 +49,25 @@ PYBIND11_MODULE(libbbf, m) {
         
         .def("get_metadata", &BBFReader::getMetadata,
              "Returns a list of (Key, Value) tuples.")
+             
+        .def("get_page_info", &BBFReader::getPageInfo,
+             "Returns dict with keys: length, offset, hash, type, decodedLength")
 
         .def("get_page_data", [](BBFReader& r, uint32_t idx) {
              auto raw = r.getPageRaw(idx);
              if (!raw.first) return py::bytes("");
-             // 1-Copy: Copies from mmap -> Python Bytes Object
              return py::bytes(raw.first, raw.second);
         }, "Returns the raw bytes of the page asset (1-Copy).")
 
         .def("get_page_view", [](BBFReader& r, uint32_t idx) {
              auto raw = r.getPageRaw(idx);
              if (!raw.first) return py::memoryview(py::bytes("")); 
              
-             // 0-Copy: Direct view into mmap
-             // Warning: This view crashes Python if BBFReader is garbage collected before the view!
-             // To fix this lifetime issue, we use 'py::keep_alive'.
              return py::memoryview::from_memory(
                  const_cast<char*>(raw.first), 
                  raw.second,
                  true // read-only
              );
-        }, py::keep_alive<0, 1>(), // Keep BBFReader (1) alive while memoryview (0) exists
+        }, py::keep_alive<0, 1>(), 
            "Returns a zero-copy memoryview of the asset. Fastest method.");
 }
diff --git a/src/libbbf.cpp b/src/libbbf.cpp
@@ -1,9 +1,11 @@
 #include "libbbf.h"
-#define XXH_INLINE_ALL 
 #include "xxhash.h"
 
 #include <iostream>
 #include <vector>
+#include <algorithm>
+#include <string>
+#include <cctype>
 
 BBFBuilder::BBFBuilder(const std::string& outputFilename) : currentOffset(0)
 {
@@ -23,7 +25,9 @@ BBFBuilder::BBFBuilder(const std::string& outputFilename) : currentOffset(0)
     header.magic[1] = 'B';
     header.magic[2] = 'F';
     header.magic[3] = '1';
-    header.version = 1;
+    header.version = 2;
+    header.flags = 0; // reserved for now as well.
+    header.headerLen = sizeof(BBFHeader);
     header.reserved = 0; // Reserved for future expansions
 
     // Write the header
@@ -93,16 +97,19 @@ bool BBFBuilder::addPage(const std::string& imagePath, uint8_t type, uint32_t fl
         // No dupe found. create a new asset.
         alignPadding(); // start by allocating necessary padding.
 
-        BBFAssetEntry newAsset;
+        BBFAssetEntry newAsset = {0};
         newAsset.offset = currentOffset;
         newAsset.length = size;
+        newAsset.decodedLength = size;
         newAsset.xxh3Hash = hash;
         newAsset.type = type;
+        newAsset.flags = 0; // no flags yet.
 
-        for ( int i = 0; i < 7; i++ )
-        {
-            newAsset.reserved[i] = 0; // Add in the reserved bytes.
-        }
+        // set reserved equal to zero
+        //newAsset.reserved[4] = {0};
+
+        // same for padding
+        //newAsset.padding[7] = {0};
 
         fileStream.write(buffer.data(), size);
         currentOffset += size;
@@ -161,8 +168,6 @@ bool BBFBuilder::addMetadata(const std::string& key, const std::string& value)
 
 bool BBFBuilder::finalize()
 {
-    uint64_t indexStart = currentOffset;
-
     // Initialize XXH3 State
     XXH3_state_t* const state = XXH3_createState();
     if (state == nullptr) return false;
@@ -179,6 +184,7 @@ bool BBFBuilder::finalize()
     //write footer
     BBFFooter footer;
     footer.stringPoolOffset = currentOffset;
+    footer.extraOffset = 0; // set the extraOffset to 0 since we aren't using it.
 
     //fileStream.write(stringPool.data(), stringPool.size());
     //currentOffset += stringPool.size();
@@ -231,4 +237,42 @@ bool BBFBuilder::finalize()
     fileStream.write(reinterpret_cast<char*>(&footer), sizeof(BBFFooter));
     fileStream.close();
     return true;
+}
+
+BBFMediaType detectTypeFromExtension(const std::string &extension) 
+{
+    std::string ext = extension;
+    std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
+
+    if (ext == ".png") return BBFMediaType::PNG;
+    if (ext == ".jpg" || ext == ".jpeg") return BBFMediaType::JPG;
+    if (ext == ".avif") return BBFMediaType::AVIF;
+    if (ext == ".webp") return BBFMediaType::WEBP;
+    if (ext == ".jxl") return BBFMediaType::JXL;
+    if (ext == ".bmp") return BBFMediaType::BMP;
+    if (ext == ".gif") return BBFMediaType::GIF;
+    if (ext == ".tiff") return BBFMediaType::TIFF;
+    
+    return BBFMediaType::UNKNOWN;
+}
+
+std::string MediaTypeToStr(uint8_t type)
+{
+    BBFMediaType mediaType = static_cast<BBFMediaType>(type);
+
+    switch (mediaType)
+    {
+        case BBFMediaType::AVIF: return ".avif";
+        case BBFMediaType::PNG:  return ".png";
+        case BBFMediaType::JPG:  return ".jpg";
+        case BBFMediaType::WEBP: return ".webp";
+        case BBFMediaType::JXL:  return ".jxl";
+        case BBFMediaType::BMP:  return ".bmp";
+        case BBFMediaType::GIF:  return ".gif";
+        case BBFMediaType::TIFF: return ".tiff";
+
+        case BBFMediaType::UNKNOWN: 
+        default: 
+            return ".png"; 
+    }
 }
diff --git a/src/libbbf.h b/src/libbbf.h
