diff --git a/.github/workflows/filo-package.yml b/.github/workflows/filo-package.yml index 43bb361..46c5a24 100644 --- a/.github/workflows/filo-package.yml +++ b/.github/workflows/filo-package.yml @@ -45,7 +45,6 @@ jobs: --api-key "${{ secrets.NUGET_KEY }}" ` --skip-duplicate # ── Publish to GitHub Packages ───────────────────────────────── - # GITHUB_TOKEN is automatically available + has package:write permission dotnet nuget push "$nupkg" ` --source "https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json" ` --api-key "${{ secrets.GITHUB_TOKEN }}" ` diff --git a/.github/workflows/filo.yml b/.github/workflows/filo.yml new file mode 100644 index 0000000..8f09f22 --- /dev/null +++ b/.github/workflows/filo.yml @@ -0,0 +1,23 @@ +name: Ytdlp.NET + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.x + - name: Restore dependencies + run: dotnet restore src/Filo/Filo.csproj + - name: Build + run: dotnet build src/Filo/Filo.csproj -c Release \ No newline at end of file diff --git a/README.md b/README.md index a7576af..dbad085 100644 --- a/README.md +++ b/README.md @@ -5,27 +5,35 @@ ![NuGet Downloads](https://img.shields.io/nuget/dt/Filo) ![Visitors](https://visitor-badge.laobi.icu/badge?page_id=manusoft/FiloContainer) ---- - FILO +## FILO v1.1 Highlights + +**Added:** + +* ✔ Chunk integrity hashing (SHA256) +* ✔ Password-based encryption (PBKDF2) + +**Deprecated:** + +* ⚠ `WithEncryption(byte[] key)` — use `WithPassword(string password)` instead. + +--- ## Overview -**FILO** (Files In, Layered & Organized) is a modern **multi-file container format** for .NET designed to handle **large files efficiently**. +**FILO** (Files In, Layered & Organized) is a modern **multi-file container format** for .NET designed for **large files**. It stores multiple files (video, audio, text, binaries, etc.) in a **single container**, supporting: -- **Large files** (videos, audio, binaries) -- **Multiple files per container** -- Chunked streaming for **GB-sized files** -- AES256 optional encryption per chunk -- Metadata storage -- File checksums for integrity -- Fully async and memory-efficient operations +* **Large files** (GB-sized videos, audio, binaries) +* **Multiple files per container** +* **Chunked streaming** for memory-efficient reads +* **AES256 optional encryption per chunk** +* **Embedded metadata** +* **File checksums** for integrity +* Fully **async APIs** -> FILO = **Files In, Layered & Organized** - -It is ideal for **video/audio streaming, backup containers, and custom file packaging**. +> FILO = **Files In, Layered & Organized** — ideal for **video/audio streaming, backups, and custom file packaging**. --- @@ -106,7 +114,7 @@ Traditional ZIP or JSON-based storage has limitations: Install via NuGet: ```bash -dotnet add package Filo.1.0.0 +dotnet add package Filo --version 1.1.0 ```` --- @@ -117,15 +125,14 @@ dotnet add package Filo.1.0.0 ```csharp using Filo; -using System.Security.Cryptography; -var key = RandomNumberGenerator.GetBytes(32); // AES256 key +var pwd = "mypassword"; var writer = new FiloWriter("backup.filo") .AddFile("video.mp4", new FileMetadata { MimeType = "video/mp4" }) .AddFile("subtitle.srt", new FileMetadata { MimeType = "text/plain" }) .WithChunkSize(5_000_000) - .WithEncryption(key); + .WithPassword(pwd); await writer.WriteAsync(); Console.WriteLine("FILO container created!"); @@ -137,51 +144,68 @@ Console.WriteLine("FILO container created!"); var reader = new FiloReader("backup.filo"); await reader.InitializeAsync(); -// List files in container +var key = reader.DeriveKey("mypassword"); + +// List files foreach (var file in reader.ListFiles()) Console.WriteLine(file); -// Stream a file (AES256 encrypted example) +// Stream chunks await foreach (var chunk in reader.StreamFileAsync("video.mp4", key)) { - // Process chunk + await chunk.WriteAsync(chunk); } ``` --- -## Streaming Video/Audio +### Direct Streaming -FILO supports **direct streaming without reassembling the file**: +```csharp +using var stream = reader.OpenStream("video.mp4", key); +await stream.CopyToAsync(outputFile); +``` + +### Extract Files Using `FiloStream` ```csharp -await using var filoStream = new FiloStream(reader, "video.mp4", key); -await using var output = new FileStream("video_streamed.mp4", FileMode.Create); +using var filoStream = new FiloStream(reader, "video.mp4", key); +using var output = File.Create("video_restored.mp4"); + await filoStream.CopyToAsync(output); +Console.WriteLine("File extracted!"); +``` + +### Load Image + +```csharp +using var stream = new FiloStream(reader, "photo.jpg"); +using var image = System.Drawing.Image.FromStream(stream); + +Console.WriteLine($"Loaded image: {image.Width}x{image.Height}"); + ``` -### In Blazor Server / ASP.NET Core +## Streaming Video/Audio in ASP.NET Core / Blazor ```csharp -[HttpGet("video/{fileName}")] -public async Task StreamVideo(string fileName) +public async Task GetVideo() { - var reader = new FiloReader("backup.filo"); + var reader = new FiloReader("media.filo"); await reader.InitializeAsync(); - var filoStream = new FiloStream(reader, fileName, key: YourKeyHere); - return File(filoStream, "video/mp4", enableRangeProcessing: true); + var key = reader.DeriveKey("password"); + var stream = new FiloStream(reader, "movie.mp4", key); + + return File(stream, "video/mp4"); } ``` -* Supports **large files**, **streaming**, and **AES256 encrypted chunks** -* Browser can **seek**, **pause**, and **resume** seamlessly +> Supports **large files**, **streaming**, and **AES256 encrypted chunks**. Browser can **seek, pause, and resume** seamlessly. --- -## Multi-file container - -You can store multiple files in the same container: +## Multi-file Container Example ```csharp var writer = new FiloWriter("media.filo") @@ -189,21 +213,21 @@ var writer = new FiloWriter("media.filo") .AddFile("audio.mp3", new FileMetadata { MimeType = "audio/mpeg" }) .AddFile("subtitle.srt", new FileMetadata { MimeType = "text/plain" }) .WithChunkSize(10_000_000) - .WithEncryption(key); + .WithPassword("mypassword"); await writer.WriteAsync(); ``` -* The container will store **indexes, metadata, and checksums**. -* You can **stream each file individually** using `FiloStream` or `StreamFileAsync`. +* Stores **indexes, metadata, and checksums** +* Stream **each file individually** using `FiloStream` or `StreamFileAsync` --- ## Chunked Streaming -* FILO reads files in **chunks** to minimize memory usage. -* Suitable for **large video/audio files**. -* Supports **AES256 encryption per chunk**. +* Reads files in **memory-efficient chunks** +* Ideal for **large video/audio files** +* Supports **AES256 encryption per chunk** ```csharp await foreach (var chunk in reader.StreamFileAsync("largevideo.mp4", key)) @@ -212,20 +236,30 @@ await foreach (var chunk in reader.StreamFileAsync("largevideo.mp4", key)) } ``` -> Always verify checksum for **large file integrity**. +--- + +## ⚡ When to Use FiloStream vs StreamFileAsync + +| Method | Best For | +| ------------------| ---------------- | +| StreamFileAsync() | chunk processing | +| FiloStream normal | file streaming | +| CopyToAsync() | extraction | +| HTTP streaming | media servers | --- -## Checksums & Integrity +> Always verify checksum for **large file integrity**. + -FILO stores **SHA256 checksums** for each file: +## Checksums & Integrity ```csharp var checksum = await FiloChecksum.ComputeFileSHA256Async("video.mp4"); Console.WriteLine(checksum); ``` -You can verify that **streamed files match the original**. +* Ensures **streamed files match the original** --- @@ -233,10 +267,10 @@ You can verify that **streamed files match the original**. | Class | Key Methods | | ------------------ | ---------------------------------------------------------------------- | -| `FiloWriter` | `.AddFile()`, `.WithChunkSize()`, `.WithEncryption()`, `.WriteAsync()` | -| `FiloReader` | `.InitializeAsync()`, `.ListFiles()`, `.StreamFileAsync()` | -| `FiloStream` | `.ReadAsync()` – supports streaming directly to players | -| `FiloChecksum` | `.ComputeFileSHA256Async()`, `.ComputeFileSHA256Async()`, `.ComputeSHA256()`, `.Verify()`,`.VerifyFileAsync()` | +| `FiloWriter` | `.AddFile()`, `AddDirectory()`, `.WithChunkSize()`, `.WithPassword()`, `.WriteAsync()` | +| `FiloReader` | `.InitializeAsync()`, `DeriveKey()`, `FileExists()`, `GetFileInfo()`, `.ListFiles()`, `.StreamFileAsync()`, `OpenStream()`, `ExtractFileAsync()`, `ExtractDirectoryAsync()`, `ReadHeaderAsync()` | +| `FiloStream` | `.ReadAsync()` – supports streaming directly to players, `Read()` | +| `FiloChecksum` | `.ComputeSHA256()`, `.ComputeSHA256Async()`, `.ComputeFileSHA256Async()`, `.ComputeFileSHA256Async()`,`.Verify()`, `VerifyFileAsync()` | | `FiloEncryption` | `.Encrypt()`, `.Decrypt()` | --- diff --git a/src/Filo/Core/FiloReader.cs b/src/Filo/Core/FiloReader.cs index 120c84b..75caaf1 100644 --- a/src/Filo/Core/FiloReader.cs +++ b/src/Filo/Core/FiloReader.cs @@ -1,4 +1,6 @@ using ManuHub.Filo.Utils; +using System.Security.Cryptography; +using System.Text; using System.Text.Json; namespace ManuHub.Filo; @@ -6,47 +8,95 @@ namespace ManuHub.Filo; public class FiloReader { private readonly string _path; + private List _fileEntries = new(); + private Dictionary _fileMap = new(StringComparer.OrdinalIgnoreCase); + public FiloHeader Header { get; private set; } = default!; public FiloReader(string path) => _path = path; /// - /// Reads the container and initializes index and metadata. + /// Initializes the container by reading header and index. /// public async Task InitializeAsync() { try { - await using var fs = new FileStream(_path, FileMode.Open, FileAccess.Read, FileShare.Read); + await using var fs = new FileStream( + _path, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + 128 * 1024, + FileOptions.SequentialScan); if (fs.Length < 16) throw new InvalidDataException("FILO container too small or corrupted."); - // Read footer for indexOffset + // MAGIC + var magicBuffer = new byte[4]; + await fs.ReadExactlyAsync(magicBuffer); + + var magic = Encoding.ASCII.GetString(magicBuffer); + + if (magic != Filo.Magic) + throw new InvalidDataException("Invalid FILO container."); + + // VERSION + var intBuffer = new byte[4]; + await fs.ReadExactlyAsync(intBuffer); + + int version = BitConverter.ToInt32(intBuffer); + + if (version > Filo.Version) + throw new InvalidDataException("Unsupported FILO version."); + + // HEADER LENGTH + await fs.ReadExactlyAsync(intBuffer); + + int headerLength = BitConverter.ToInt32(intBuffer); + + if (headerLength <= 0 || headerLength > 1_000_000) + throw new InvalidDataException("Invalid header length."); + + // HEADER + var headerBytes = new byte[headerLength]; + await fs.ReadExactlyAsync(headerBytes); + + Header = JsonSerializer.Deserialize(headerBytes) + ?? throw new InvalidDataException("Failed to deserialize header."); + + // FOOTER fs.Seek(-16, SeekOrigin.End); + var longBuffer = new byte[8]; + await fs.ReadExactlyAsync(longBuffer); long indexOffset = BitConverter.ToInt64(longBuffer); await fs.ReadExactlyAsync(longBuffer); long metadataOffset = BitConverter.ToInt64(longBuffer); - if (indexOffset >= fs.Length || metadataOffset >= fs.Length) - throw new InvalidDataException("FILO container footer offsets are invalid."); + if (indexOffset >= fs.Length) + throw new InvalidDataException("Invalid index offset."); - // Read index + // READ INDEX fs.Position = indexOffset; - var intBuffer = new byte[4]; + await fs.ReadExactlyAsync(intBuffer); int indexLen = BitConverter.ToInt32(intBuffer); if (indexLen <= 0 || indexLen > fs.Length - indexOffset) - throw new InvalidDataException("FILO container index length is invalid."); + throw new InvalidDataException("Invalid index length."); var indexBytes = new byte[indexLen]; await fs.ReadExactlyAsync(indexBytes); + _fileEntries = JsonSerializer.Deserialize>(indexBytes) - ?? throw new InvalidDataException("Failed to deserialize file index."); + ?? throw new InvalidDataException("Failed to parse index."); + + // FAST LOOKUP MAP + _fileMap = _fileEntries.ToDictionary(f => f.Path, StringComparer.OrdinalIgnoreCase); } catch (FileNotFoundException) { @@ -65,21 +115,96 @@ public async Task InitializeAsync() } } + public bool FileExists(string path) + { + return _fileMap.ContainsKey(path); + } + /// - /// Lists all files contained in the FILO container. + /// Returns file inside the container. /// - public IEnumerable ListFiles() => _fileEntries.Select(f => f.FileName); + public FiloFileInfo? GetFileInfo(string path) + { + if (!_fileMap.TryGetValue(path, out var entry)) + return null; + if (entry == null) + return null; + + return new FiloFileInfo + { + Name = Path.GetFileName(path), + Path = entry.Path, + MimeType = entry.MimeType, + FileSize = entry.FileSize, + ChunkCount = entry.Chunks.Count, + Encrypted = entry.Encrypted + }; + } + + /// + /// Returns all files inside the container. + /// + /// + public IReadOnlyList ListFiles() + { + return _fileEntries.Select(e => new FiloFileInfo + { + Name = Path.GetFileName(e.Path), + Path = e.Path, + MimeType = e.MimeType, + FileSize = e.FileSize, + ChunkCount = e.Chunks.Count, + Encrypted = e.Encrypted + }).ToList(); + } /// - /// Lists all files contained in the FILO container. + /// Derives a 256-bit AES key from password using PBKDF2. + /// + public byte[] DeriveKey(string password) + { + if (Header.Salt == null) + throw new InvalidOperationException("Container is not password protected."); + + var salt = Convert.FromBase64String(Header.Salt); + + byte[] key = new byte[32]; + + Rfc2898DeriveBytes.Pbkdf2(password, salt, key, 100_000, HashAlgorithmName.SHA256); + + if (Header.PasswordCheck != null) + { + var check = SHA256.HashData(key); + + if (!check.SequenceEqual(Header.PasswordCheck)) + throw new CryptographicException("Invalid password."); + } + + return key; + } + + /// + /// Streams a file from the container chunk-by-chunk. /// public async IAsyncEnumerable StreamFileAsync(string fileName, byte[]? key = null) { - var entry = _fileEntries.FirstOrDefault(f => f.FileName == fileName) - ?? throw new FileNotFoundException($"File '{fileName}' not found in container."); + if (!_fileMap.TryGetValue(fileName, out var entry)) + throw new FileNotFoundException($"File '{fileName}' not found in container."); + + if (Header.Encryption == "AES256" && key == null) + throw new InvalidOperationException("This container is encrypted. Provide a key."); - await using var fs = new FileStream(_path, FileMode.Open, FileAccess.Read); + if (entry.Chunks.Count == 0) + yield break; + + await using var fs = new FileStream( + _path, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + 128 * 1024, + FileOptions.SequentialScan); foreach (var chunk in entry.Chunks) { @@ -89,9 +214,8 @@ public async IAsyncEnumerable StreamFileAsync(string fileName, byte[]? k try { - if (key != null) + if (Header.Encryption == "AES256") { - // Encrypted chunk var iv = new byte[16]; await fs.ReadExactlyAsync(iv); @@ -102,18 +226,24 @@ public async IAsyncEnumerable StreamFileAsync(string fileName, byte[]? k var enc = new byte[len]; await fs.ReadExactlyAsync(enc); - dataChunk = FiloEncryption.Decrypt(enc, key, iv); + dataChunk = FiloEncryption.Decrypt(enc, key!, iv); } else { - // Plain chunk var lenBuf = new byte[4]; await fs.ReadExactlyAsync(lenBuf); + int len = BitConverter.ToInt32(lenBuf); dataChunk = new byte[len]; await fs.ReadExactlyAsync(dataChunk); } + + // INTEGRITY CHECK + var computed = Convert.ToHexString(SHA256.HashData(dataChunk)); + + if (!string.Equals(computed, chunk.Hash, StringComparison.OrdinalIgnoreCase)) + throw new InvalidDataException($"Chunk {chunk.Id} failed integrity check."); } catch (IOException ioEx) { @@ -129,4 +259,59 @@ public async IAsyncEnumerable StreamFileAsync(string fileName, byte[]? k yield return dataChunk; // must be outside try/catch } } + + /// + /// Creates a stream for reading a file directly. + /// + public Stream OpenStream(string fileName, byte[]? key = null) => new FiloStream(this, fileName, key); + + public async Task ExtractFileAsync(string path, string outputPath, byte[]? key = null) + { + using var stream = OpenStream(path, key); + using var output = File.Create(outputPath); + + await stream.CopyToAsync(output); + } + + public async Task ExtractDirectoryAsync(string directory, string outputFolder, byte[]? key = null) + { + Directory.CreateDirectory(outputFolder); + + var files = ListFiles() + .Where(f => f.Path.StartsWith(directory, StringComparison.OrdinalIgnoreCase)); + + foreach (var file in files) + { + var relative = file.Path.Substring(directory.Length).TrimStart('/'); + + var outputPath = Path.Combine(outputFolder, relative); + + Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!); + + await using var input = new FiloStream(this, file.Path, key); + await using var output = new FileStream(outputPath, FileMode.Create); + + await input.CopyToAsync(output); + } + } + + public static async Task ReadHeaderAsync(string path) + { + await using var fs = new FileStream(path, FileMode.Open, FileAccess.Read); + + var magic = new byte[4]; + await fs.ReadExactlyAsync(magic); + + var intBuf = new byte[4]; + + await fs.ReadExactlyAsync(intBuf); // version + + await fs.ReadExactlyAsync(intBuf); + int headerLen = BitConverter.ToInt32(intBuf); + + var headerBytes = new byte[headerLen]; + await fs.ReadExactlyAsync(headerBytes); + + return JsonSerializer.Deserialize(headerBytes)!; + } } \ No newline at end of file diff --git a/src/Filo/Core/FiloStream.cs b/src/Filo/Core/FiloStream.cs index 602b8a9..453f571 100644 --- a/src/Filo/Core/FiloStream.cs +++ b/src/Filo/Core/FiloStream.cs @@ -5,8 +5,12 @@ public class FiloStream : Stream private readonly FiloReader _reader; private readonly string _fileName; private readonly byte[]? _key; + private IAsyncEnumerator? _chunks; - private MemoryStream? _currentChunk; + + private byte[]? _currentChunk; + private int _chunkPosition; + private bool _initialized; public FiloStream(FiloReader reader, string fileName, byte[]? key = null) @@ -20,31 +24,44 @@ public FiloStream(FiloReader reader, string fileName, byte[]? key = null) public override bool CanSeek => false; public override bool CanWrite => false; public override long Length => throw new NotSupportedException(); - public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } public override void Flush() => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); private async Task EnsureInitializedAsync() { - if (!_initialized) - { - _chunks = _reader.StreamFileAsync(_fileName, _key).GetAsyncEnumerator(); - _initialized = true; - await MoveNextChunkAsync(); - } + if (_initialized) + return; + + _chunks = _reader.StreamFileAsync(_fileName, _key).GetAsyncEnumerator(); + _initialized = true; + + await MoveNextChunkAsync(); } private async Task MoveNextChunkAsync() { - if (_chunks != null) + if (_chunks == null) + return; + + if (await _chunks.MoveNextAsync()) { - if (await _chunks.MoveNextAsync()) - _currentChunk = new MemoryStream(_chunks.Current); - else - _currentChunk = null; + _currentChunk = _chunks.Current; + _chunkPosition = 0; + } + else + { + _currentChunk = null; } } @@ -53,33 +70,53 @@ public override async Task ReadAsync(byte[] buffer, int offset, int count, await EnsureInitializedAsync(); if (_currentChunk == null) - return 0; // End of stream + return 0; int totalRead = 0; while (count > 0 && _currentChunk != null) { - int read = await _currentChunk.ReadAsync(buffer, offset, count, cancellationToken); - totalRead += read; - offset += read; - count -= read; + int remaining = _currentChunk.Length - _chunkPosition; - if (_currentChunk.Position >= _currentChunk.Length) + if (remaining <= 0) + { await MoveNextChunkAsync(); + continue; + } + + int toCopy = Math.Min(count, remaining); + + Buffer.BlockCopy(_currentChunk, _chunkPosition, buffer, offset, toCopy); + + _chunkPosition += toCopy; + offset += toCopy; + count -= toCopy; + totalRead += toCopy; } return totalRead; } - protected override void Dispose(bool disposing) + public override int Read(byte[] buffer, int offset, int count) { - _chunks?.DisposeAsync().AsTask().Wait(); - _currentChunk?.Dispose(); - base.Dispose(disposing); + return ReadAsync(buffer, offset, count) + .GetAwaiter() + .GetResult(); } - public override int Read(byte[] buffer, int offset, int count) + protected override void Dispose(bool disposing) { - throw new NotImplementedException(); + if (disposing) + { + if (_chunks != null) + { + _chunks.DisposeAsync() + .AsTask() + .GetAwaiter() + .GetResult(); + } + } + + base.Dispose(disposing); } } \ No newline at end of file diff --git a/src/Filo/Core/FiloWriter.cs b/src/Filo/Core/FiloWriter.cs index 1321387..bb63c35 100644 --- a/src/Filo/Core/FiloWriter.cs +++ b/src/Filo/Core/FiloWriter.cs @@ -11,30 +11,93 @@ public class FiloWriter private readonly List<(string path, FileMetadata meta)> _files = new(); private int _chunkSize = 10_000_000; private bool _encrypt; - private byte[]? _key; + private string? _password; + private byte[]? _key = new byte[32]; + private byte[]? _salt; public FiloWriter(string outputPath) => _outputPath = outputPath; public FiloWriter AddFile(string filePath, FileMetadata metadata) { + var containerPath = metadata.ContainerPath ?? Path.GetFileName(filePath); + + if (_files.Any(f => + (f.meta.ContainerPath ?? Path.GetFileName(f.path)) + .Equals(containerPath, StringComparison.OrdinalIgnoreCase))) + { + throw new InvalidOperationException($"Duplicate container path: {containerPath}"); + } + _files.Add((filePath, metadata)); + + return this; + } + + public FiloWriter AddDirectory(string directoryPath, string? containerRoot = null) + { + if (!Directory.Exists(directoryPath)) + throw new DirectoryNotFoundException(directoryPath); + + var root = new DirectoryInfo(directoryPath); + + foreach (var file in root.GetFiles("*", SearchOption.AllDirectories)) + { + var relative = Path.GetRelativePath(directoryPath, file.FullName) + .Replace('\\', '/'); + + var containerPath = containerRoot == null + ? relative + : $"{containerRoot.TrimEnd('/')}/{relative}"; + + AddFile(file.FullName, new FileMetadata + { + MimeType = GuessMimeType(file.FullName), + ContainerPath = containerPath + }); + } + + return this; + } + + public FiloWriter WithChunkSize(int chunkSize) + { + _chunkSize = chunkSize; + return this; + } + + public FiloWriter WithPassword(string password) + { + _encrypt = true; + _password = password; return this; } - public FiloWriter WithChunkSize(int chunkSize) { _chunkSize = chunkSize; return this; } - public FiloWriter WithEncryption(byte[] key) { _encrypt = true; _key = key; return this; } + [Obsolete("Use WithPassword(string password) instead. Raw key support will be removed in FILO v2.0.")] + public FiloWriter WithEncryption(byte[] key) + { + _encrypt = true; + _key = key; + return this; + } public async Task WriteAsync() { if (_files.Count == 0) throw new InvalidOperationException("No files added to the container."); + if (_encrypt && _password != null) + { + DeriveKeyFromPassword(); + } + try { await using var output = new FileStream(_outputPath, FileMode.Create); - // MAGIC + VERSION + // MAGIC await output.WriteAsync(Encoding.ASCII.GetBytes(Filo.Magic)); + + // VERSION await output.WriteAsync(BitConverter.GetBytes(Filo.Version)); // HEADER @@ -47,11 +110,15 @@ public async Task WriteAsync() FileCount = _files.Count, Compression = "none", Encryption = _encrypt ? "AES256" : "none", + Kdf = !string.IsNullOrWhiteSpace(_password) ? "PBKDF2" : null, + Salt = !string.IsNullOrWhiteSpace(_password) ? Convert.ToBase64String(_salt!) : null, + PasswordCheck = _encrypt ? SHA256.HashData(_key!) : null, Description = "FILO multi-file container" }; var headerJson = JsonSerializer.Serialize(header); var headerBytes = Encoding.UTF8.GetBytes(headerJson); + await output.WriteAsync(BitConverter.GetBytes(headerBytes.Length)); await output.WriteAsync(headerBytes); @@ -62,9 +129,10 @@ public async Task WriteAsync() { var entry = new FileEntry { - FileName = Path.GetFileName(filePath), + Path = meta.ContainerPath ?? Path.GetFileName(filePath), MimeType = meta.MimeType, - FileSize = new FileInfo(filePath).Length + FileSize = new FileInfo(filePath).Length, + Encrypted = _encrypt, }; try @@ -80,22 +148,37 @@ public async Task WriteAsync() var offset = output.Position; byte[] chunk = buffer[..read]; + string hash = Convert.ToHexString(SHA256.HashData(chunk)); + if (_encrypt) { var iv = RandomNumberGenerator.GetBytes(16); var encrypted = FiloEncryption.Encrypt(chunk, _key!, iv); + await output.WriteAsync(iv); await output.WriteAsync(BitConverter.GetBytes(encrypted.Length)); await output.WriteAsync(encrypted); - entry.Chunks.Add(new FiloChunkIndex { Id = chunkId++, Offset = offset, Length = encrypted.Length }); + entry.Chunks.Add(new FiloChunkIndex + { + Id = chunkId++, + Offset = offset, + Length = encrypted.Length, + Hash = hash + }); } else { await output.WriteAsync(BitConverter.GetBytes(read)); await output.WriteAsync(chunk); - entry.Chunks.Add(new FiloChunkIndex { Id = chunkId++, Offset = offset, Length = read }); + entry.Chunks.Add(new FiloChunkIndex + { + Id = chunkId++, + Offset = offset, + Length = read, + Hash = hash + }); } } } @@ -104,6 +187,7 @@ public async Task WriteAsync() Console.Error.WriteLine($"Error reading file '{filePath}': {ioEx.Message}"); throw; } + fileEntries.Add(entry); } @@ -141,4 +225,30 @@ public async Task WriteAsync() throw; } } + + private void DeriveKeyFromPassword() + { + _salt = RandomNumberGenerator.GetBytes(32); + _key = new byte[32]; + Rfc2898DeriveBytes.Pbkdf2(_password!, _salt, _key, 100_000, HashAlgorithmName.SHA256); + } + + private static string GuessMimeType(string file) + { + var ext = Path.GetExtension(file).ToLowerInvariant(); + + return ext switch + { + ".mp4" => "video/mp4", + ".mkv" => "video/x-matroska", + ".mp3" => "audio/mpeg", + ".wav" => "audio/wav", + ".srt" => "text/plain", + ".txt" => "text/plain", + ".json" => "application/json", + ".jpg" => "image/jpeg", + ".png" => "image/png", + _ => "application/octet-stream" + }; + } } \ No newline at end of file diff --git a/src/Filo/Filo.csproj b/src/Filo/Filo.csproj index c4efb8a..c842049 100644 --- a/src/Filo/Filo.csproj +++ b/src/Filo/Filo.csproj @@ -6,11 +6,10 @@ enable latest Filo - 1.0.0.0 - 1.0.0.0 + 1.1.0 ManuHub.Filo Filo - 1.0.0 + 1.1.0 Manojbabu ManuHub FILO – Fast, flexible multi-file container for .NET with streaming, encryption, and metadata support. @@ -23,12 +22,9 @@ LICENSE.txt icon.png icon.png - true - 1.0.0 Initial release with multi-file streaming, chunked encryption, and metadata support. false - true true diff --git a/src/Filo/Models/FileEntry.cs b/src/Filo/Models/FileEntry.cs index 10b8b2d..444facb 100644 --- a/src/Filo/Models/FileEntry.cs +++ b/src/Filo/Models/FileEntry.cs @@ -2,8 +2,9 @@ public class FileEntry { - public string FileName { get; set; } = ""; + public string Path { get; set; } = ""; public string MimeType { get; set; } = ""; public long FileSize { get; set; } + public bool Encrypted { get; set; } public List Chunks { get; set; } = new(); -} \ No newline at end of file +} diff --git a/src/Filo/Models/FileList.cs b/src/Filo/Models/FileList.cs new file mode 100644 index 0000000..9d34c92 --- /dev/null +++ b/src/Filo/Models/FileList.cs @@ -0,0 +1,9 @@ +namespace ManuHub.Filo; + +public class FileList +{ + public string FileName { get; set; } = ""; + public string MimeType { get; set; } = ""; + public long FileSize { get; set; } + public bool Encrypted { get; set; } +} \ No newline at end of file diff --git a/src/Filo/Models/FileMetadata.cs b/src/Filo/Models/FileMetadata.cs index fb693b0..fe7b7b5 100644 --- a/src/Filo/Models/FileMetadata.cs +++ b/src/Filo/Models/FileMetadata.cs @@ -4,5 +4,6 @@ public class FileMetadata { public string MimeType { get; set; } = ""; public string Description { get; set; } = ""; + public string? ContainerPath { get; set; } public Dictionary Tags { get; set; } = new(); } diff --git a/src/Filo/Models/FiloChunkIndex.cs b/src/Filo/Models/FiloChunkIndex.cs index e68d1e3..51239d1 100644 --- a/src/Filo/Models/FiloChunkIndex.cs +++ b/src/Filo/Models/FiloChunkIndex.cs @@ -5,4 +5,5 @@ public class FiloChunkIndex public int Id { get; set; } public long Offset { get; set; } public int Length { get; set; } + public string? Hash { get; set; } // v1.1 } \ No newline at end of file diff --git a/src/Filo/Models/FiloFileInfo.cs b/src/Filo/Models/FiloFileInfo.cs new file mode 100644 index 0000000..69e21b1 --- /dev/null +++ b/src/Filo/Models/FiloFileInfo.cs @@ -0,0 +1,13 @@ +namespace ManuHub.Filo; + +public class FiloFileInfo +{ + public string Name { get; set; } = ""; + public string Path { get; set; } = ""; + public string MimeType { get; set; } = ""; + public long FileSize { get; set; } + public int ChunkCount { get; set; } + public bool Encrypted { get; set; } + + public string Directory => System.IO.Path.GetDirectoryName(Path)?.Replace('\\', '/') ?? ""; +} \ No newline at end of file diff --git a/src/Filo/Models/FiloHeader.cs b/src/Filo/Models/FiloHeader.cs index 078548e..cea7efe 100644 --- a/src/Filo/Models/FiloHeader.cs +++ b/src/Filo/Models/FiloHeader.cs @@ -2,43 +2,23 @@ public class FiloHeader { - /// - /// Container format name - /// public string Format { get; set; } = Filo.Magic; - - /// - /// Format version - /// public int Version { get; set; } = 1; - - /// - /// When the container was created - /// public DateTime Created { get; set; } = DateTime.UtcNow; - /// - /// Chunk size used for binary storage - /// public int ChunkSize { get; set; } - /// - /// Number of files stored in the container - /// - public int FileCount { get; set; } = 0; + public int FileCount { get; set; } = 0; - /// - /// Compression algorithm (none, gzip, zstd etc.) - /// public string Compression { get; set; } = "none"; - /// - /// Encryption algorithm (none, aes256 etc.) - /// public string Encryption { get; set; } = "none"; - /// - /// Optional container description - /// + public string? Kdf { get; set; } // v1.1 + + public string? Salt { get; set; } // v1.1 + + public byte[]? PasswordCheck { get; set; } + public string? Description { get; set; } } \ No newline at end of file diff --git a/src/Filo/README.md b/src/Filo/README.md index 937ca17..3279500 100644 --- a/src/Filo/README.md +++ b/src/Filo/README.md @@ -6,22 +6,33 @@ --- +## FILO v1.1 Highlights + +**Added:** + +* ✔ Chunk integrity hashing (SHA256) +* ✔ Password-based encryption (PBKDF2) + +**Deprecated:** + +* ⚠ `WithEncryption(byte[] key)` — use `WithPassword(string password)` instead. + +--- + ## Overview -**FILO** (Files In, Layered & Organized) is a modern **multi-file container format** for .NET designed to handle **large files efficiently**. +**FILO** (Files In, Layered & Organized) is a modern **multi-file container format** for .NET designed for **large files**. It stores multiple files (video, audio, text, binaries, etc.) in a **single container**, supporting: -- **Large files** (videos, audio, binaries) -- **Multiple files per container** -- Chunked streaming for **GB-sized files** -- AES256 optional encryption per chunk -- Metadata storage -- File checksums for integrity -- Fully async and memory-efficient operations - -> FILO = **Files In, Layered & Organized** +* **Large files** (GB-sized videos, audio, binaries) +* **Multiple files per container** +* **Chunked streaming** for memory-efficient reads +* **AES256 optional encryption per chunk** +* **Embedded metadata** +* **File checksums** for integrity +* Fully **async APIs** -It is ideal for **video/audio streaming, backup containers, and custom file packaging**. +> FILO = **Files In, Layered & Organized** — ideal for **video/audio streaming, backups, and custom file packaging**. --- @@ -102,7 +113,7 @@ Traditional ZIP or JSON-based storage has limitations: Install via NuGet: ```bash -dotnet add package Filo.1.0.0 +dotnet add package Filo --version 1.1.0 ```` --- @@ -113,15 +124,14 @@ dotnet add package Filo.1.0.0 ```csharp using Filo; -using System.Security.Cryptography; -var key = RandomNumberGenerator.GetBytes(32); // AES256 key +var pwd = "mypassword"; var writer = new FiloWriter("backup.filo") .AddFile("video.mp4", new FileMetadata { MimeType = "video/mp4" }) .AddFile("subtitle.srt", new FileMetadata { MimeType = "text/plain" }) .WithChunkSize(5_000_000) - .WithEncryption(key); + .WithPassword(pwd); await writer.WriteAsync(); Console.WriteLine("FILO container created!"); @@ -133,51 +143,68 @@ Console.WriteLine("FILO container created!"); var reader = new FiloReader("backup.filo"); await reader.InitializeAsync(); -// List files in container +var key = reader.DeriveKey("mypassword"); + +// List files foreach (var file in reader.ListFiles()) Console.WriteLine(file); -// Stream a file (AES256 encrypted example) +// Stream chunks await foreach (var chunk in reader.StreamFileAsync("video.mp4", key)) { - // Process chunk + await chunk.WriteAsync(chunk); } ``` --- -## Streaming Video/Audio +### Direct Streaming + +```csharp +using var stream = reader.OpenStream("video.mp4", key); +await stream.CopyToAsync(outputFile); +``` -FILO supports **direct streaming without reassembling the file**: +### Extract Files Using `FiloStream` ```csharp -await using var filoStream = new FiloStream(reader, "video.mp4", key); -await using var output = new FileStream("video_streamed.mp4", FileMode.Create); +using var filoStream = new FiloStream(reader, "video.mp4", key); +using var output = File.Create("video_restored.mp4"); + await filoStream.CopyToAsync(output); +Console.WriteLine("File extracted!"); ``` -### In Blazor Server / ASP.NET Core +### Load Image ```csharp -[HttpGet("video/{fileName}")] -public async Task StreamVideo(string fileName) +using var stream = new FiloStream(reader, "photo.jpg"); +using var image = System.Drawing.Image.FromStream(stream); + +Console.WriteLine($"Loaded image: {image.Width}x{image.Height}"); + +``` + +## Streaming Video/Audio in ASP.NET Core / Blazor + +```csharp +public async Task GetVideo() { - var reader = new FiloReader("backup.filo"); + var reader = new FiloReader("media.filo"); await reader.InitializeAsync(); - var filoStream = new FiloStream(reader, fileName, key: YourKeyHere); - return File(filoStream, "video/mp4", enableRangeProcessing: true); + var key = reader.DeriveKey("password"); + var stream = new FiloStream(reader, "movie.mp4", key); + + return File(stream, "video/mp4"); } ``` -* Supports **large files**, **streaming**, and **AES256 encrypted chunks** -* Browser can **seek**, **pause**, and **resume** seamlessly +> Supports **large files**, **streaming**, and **AES256 encrypted chunks**. Browser can **seek, pause, and resume** seamlessly. --- -## Multi-file container - -You can store multiple files in the same container: +## Multi-file Container Example ```csharp var writer = new FiloWriter("media.filo") @@ -185,21 +212,21 @@ var writer = new FiloWriter("media.filo") .AddFile("audio.mp3", new FileMetadata { MimeType = "audio/mpeg" }) .AddFile("subtitle.srt", new FileMetadata { MimeType = "text/plain" }) .WithChunkSize(10_000_000) - .WithEncryption(key); + .WithPassword("mypassword"); await writer.WriteAsync(); ``` -* The container will store **indexes, metadata, and checksums**. -* You can **stream each file individually** using `FiloStream` or `StreamFileAsync`. +* Stores **indexes, metadata, and checksums** +* Stream **each file individually** using `FiloStream` or `StreamFileAsync` --- ## Chunked Streaming -* FILO reads files in **chunks** to minimize memory usage. -* Suitable for **large video/audio files**. -* Supports **AES256 encryption per chunk**. +* Reads files in **memory-efficient chunks** +* Ideal for **large video/audio files** +* Supports **AES256 encryption per chunk** ```csharp await foreach (var chunk in reader.StreamFileAsync("largevideo.mp4", key)) @@ -208,20 +235,30 @@ await foreach (var chunk in reader.StreamFileAsync("largevideo.mp4", key)) } ``` -> Always verify checksum for **large file integrity**. +--- + +## ⚡ When to Use FiloStream vs StreamFileAsync + +| Method | Best For | +| ------------------| ---------------- | +| StreamFileAsync() | chunk processing | +| FiloStream normal | file streaming | +| CopyToAsync() | extraction | +| HTTP streaming | media servers | --- -## Checksums & Integrity +> Always verify checksum for **large file integrity**. + -FILO stores **SHA256 checksums** for each file: +## Checksums & Integrity ```csharp var checksum = await FiloChecksum.ComputeFileSHA256Async("video.mp4"); Console.WriteLine(checksum); ``` -You can verify that **streamed files match the original**. +* Ensures **streamed files match the original** --- @@ -229,10 +266,10 @@ You can verify that **streamed files match the original**. | Class | Key Methods | | ------------------ | ---------------------------------------------------------------------- | -| `FiloWriter` | `.AddFile()`, `.WithChunkSize()`, `.WithEncryption()`, `.WriteAsync()` | -| `FiloReader` | `.InitializeAsync()`, `.ListFiles()`, `.StreamFileAsync()` | -| `FiloStream` | `.ReadAsync()` – supports streaming directly to players | -| `FiloChecksum` | `.ComputeFileSHA256Async()`, `.ComputeFileSHA256Async()`, `.ComputeSHA256()`, `.Verify()`,`.VerifyFileAsync()` | +| `FiloWriter` | `.AddFile()`, `AddDirectory()`, `.WithChunkSize()`, `.WithPassword()`, `.WriteAsync()` | +| `FiloReader` | `.InitializeAsync()`, `DeriveKey()`, `FileExists()`, `GetFileInfo()`, `.ListFiles()`, `.StreamFileAsync()`, `OpenStream()`, `ExtractFileAsync()`, `ExtractDirectoryAsync()`, `ReadHeaderAsync()` | +| `FiloStream` | `.ReadAsync()` – supports streaming directly to players, `Read()` | +| `FiloChecksum` | `.ComputeSHA256()`, `.ComputeSHA256Async()`, `.ComputeFileSHA256Async()`, `.ComputeFileSHA256Async()`,`.Verify()`, `VerifyFileAsync()` | | `FiloEncryption` | `.Encrypt()`, `.Decrypt()` | --- diff --git a/src/Filo/Utils/FiloEncryption.cs b/src/Filo/Utils/FiloEncryption.cs index 80f3e8d..f110122 100644 --- a/src/Filo/Utils/FiloEncryption.cs +++ b/src/Filo/Utils/FiloEncryption.cs @@ -1,29 +1,34 @@ using System.Security.Cryptography; -using System.Text; namespace ManuHub.Filo.Utils; public static class FiloEncryption { - private static readonly byte[] DefaultKey = Encoding.UTF8.GetBytes("12345678901234561234567890123456"); - public static byte[] Encrypt(byte[] data, byte[] key, byte[] iv) { using var aes = Aes.Create(); + aes.Key = key; aes.IV = iv; + aes.Mode = CipherMode.CBC; + aes.Padding = PaddingMode.PKCS7; using var encryptor = aes.CreateEncryptor(); + return encryptor.TransformFinalBlock(data, 0, data.Length); } public static byte[] Decrypt(byte[] data, byte[] key, byte[] iv) { using var aes = Aes.Create(); + aes.Key = key; aes.IV = iv; + aes.Mode = CipherMode.CBC; + aes.Padding = PaddingMode.PKCS7; using var decryptor = aes.CreateDecryptor(); + return decryptor.TransformFinalBlock(data, 0, data.Length); } } \ No newline at end of file diff --git a/test/Filo.Console/Program.cs b/test/Filo.Console/Program.cs index 0db514b..2134ade 100644 --- a/test/Filo.Console/Program.cs +++ b/test/Filo.Console/Program.cs @@ -1,35 +1,67 @@ using ManuHub.Filo; using System.Security.Cryptography; -string filoPath = "backup.filo"; -byte[] key = RandomNumberGenerator.GetBytes(32); // AES256 key - -// Create container -var writer = new FiloWriter(filoPath) - .AddFile("C:\\Users\\manua\\Videos\\anu.mp4", new FileMetadata { MimeType = "video/mp4" }) - .AddFile("C:\\Users\\manua\\Videos\\numb.mp4", new FileMetadata { MimeType = "video/mp4" }) - .AddFile("C:\\Users\\manua\\Videos\\psy.mp4", new FileMetadata { MimeType = "video/mp4" }) - .AddFile("C:\\Users\\manua\\Videos\\rick.mp4", new FileMetadata { MimeType = "video/mp4" }) - .WithChunkSize(5_000_000); - //.WithEncryption(key); - -await writer.WriteAsync(); -Console.WriteLine("FILO container written!"); - -// Read container -var reader = new FiloReader(filoPath); -await reader.InitializeAsync(); - -Console.WriteLine("Files in container:"); -foreach (var f in reader.ListFiles()) Console.WriteLine(f); - -// Reassemble -foreach (var f in reader.ListFiles()) +string filoPath = "backupv1.1.filo"; + +try +{ + Console.WriteLine("Create new filo container:"); + // Create container + var writer = new FiloWriter(filoPath) + .AddFile("C:\\Users\\manua\\Videos\\anu.mp4", new FileMetadata { MimeType = "video/mp4" }) + .AddFile("C:\\Users\\manua\\Videos\\anu_kavya.mp4", new FileMetadata { MimeType = "video/mp4" }) + .WithChunkSize(5_000_000) + .WithPassword("1234567890"); + + await writer.WriteAsync(); + Console.WriteLine("FILO container written!"); +} +catch (CryptographicException cex) { - string outFile = $"restored_{f}"; - await using var filoStream = new FiloStream(reader, f); - await using var output = new FileStream(outFile, FileMode.Create); - await filoStream.CopyToAsync(output); + Console.WriteLine($"CRYPTO ERROR: {cex.Message}"); + return; } +catch (Exception ex) +{ + Console.WriteLine($"ERROR:{ex.Message}"); + return; +} + +try +{ + Console.WriteLine("Read file header:"); + var header = await FiloReader.ReadHeaderAsync("backupv1.1.filo"); -Console.WriteLine("All files reassembled successfully!"); \ No newline at end of file + Console.WriteLine($"Files: {header.FileCount}"); + Console.WriteLine($"Created: {header.Created}"); + Console.WriteLine(); + + // Read container + var reader = new FiloReader(filoPath); + await reader.InitializeAsync(); + var key = reader.DeriveKey("1234567890"); + + Console.WriteLine("Files in container:"); + // List files in container + foreach (var file in reader.ListFiles()) + Console.WriteLine($"{file.Name} ({file.FileSize} bytes)"); + + // Reassemble + foreach (var f in reader.ListFiles()) + { + string outFile = $"restored_{f.Name}"; + await using var filoStream = new FiloStream(reader, f.Path, key); + await using var output = new FileStream(outFile, FileMode.Create); + await filoStream.CopyToAsync(output); + } + + Console.WriteLine("All files reassembled successfully!"); +} +catch (CryptographicException cex) +{ + Console.WriteLine($"CRYPTO ERROR: {cex.Message}"); +} +catch (Exception ex) +{ + Console.WriteLine($"ERROR:{ex.Message}"); +} \ No newline at end of file