From da5b272e4b1d2d235ae36f8c7757b822bbe3e17d Mon Sep 17 00:00:00 2001 From: Martin Ashby Date: Mon, 18 Sep 2023 21:14:31 +0100 Subject: More work on writing zip but it doesn't work yet --- src/bar.txt | 10 ++- src/foo.txt | 2 +- src/foo/foo.txt | 2 +- src/main.zig | 246 ++++++++++++++++++++++++++++++--------------------- test/foo/foo.txt | 1 - test_out.zip | Bin 0 -> 218 bytes test_out/src/bar.txt | 0 test_out/src/foo.txt | 0 8 files changed, 157 insertions(+), 104 deletions(-) delete mode 100644 test/foo/foo.txt create mode 100644 test_out.zip create mode 100644 test_out/src/bar.txt create mode 100644 test_out/src/foo.txt diff --git a/src/bar.txt b/src/bar.txt index adabbec..35859b5 100644 --- a/src/bar.txt +++ b/src/bar.txt @@ -1 +1,9 @@ -hello there \ No newline at end of file +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. + + + + + + + +FOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO \ No newline at end of file diff --git a/src/foo.txt b/src/foo.txt index decec2d..08e00ed 100644 --- a/src/foo.txt +++ b/src/foo.txt @@ -1 +1 @@ - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. \ No newline at end of file +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. \ No newline at end of file diff --git a/src/foo/foo.txt b/src/foo/foo.txt index decec2d..08e00ed 100644 --- a/src/foo/foo.txt +++ b/src/foo/foo.txt @@ -1 +1 @@ - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. \ No newline at end of file +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. \ No newline at end of file diff --git a/src/main.zig b/src/main.zig index 0d33600..71def8f 100644 --- a/src/main.zig +++ b/src/main.zig @@ -171,30 +171,30 @@ fn pump(reader: anytype, writer: anytype, expected_size_written: usize, expected if (crc32.final() != expected_crc32) return error.WrongChecksum; } -fn pump_returning(reader: anytype, writer: anytype) !struct { written: usize, crc32: u32 } { +fn pump_returning(reader: anytype, writer: anytype) !struct { u64, u32 } { + var cw = std.io.countingWriter(writer); + var cww = cw.writer(); var buf = [_]u8{0} ** 1024; var crc32 = std.hash.Crc32.init(); - var written: usize = 0; while (true) { + std.log.err("reading...", .{}); const read = try reader.read(&buf); if (read == 0) break; const write = buf[0..read]; - try writer.writeAll(write); + try cww.writeAll(write); crc32.update(write); - written += read; } - return .{ - .written = written, - .crc32 = crc32.final(), - }; + const res = .{cw.bytes_written,crc32.final()}; + std.log.err("res... {any}", .{res}); + return res; } const CentralDirectoryHeader = struct { const SIG: u32 = @as(u32, 0x02014b50); version_made_by: u16, version_needed_to_extract: u16, - general_purpose_bit_flag: u16, + general_purpose_bit_flag: std.bit_set.IntegerBitSet(16), compression_method: CompressionMethod, last_mod_file_time: u16, last_mod_file_date: u16, @@ -224,12 +224,12 @@ const CentralDirectoryHeader = struct { // TODO generics self.file_name_length = @intCast(self.file_name.len); self.extra_field_length = @intCast(self.extra_field.len); - self.file_comment_length = @intCast(self.file_comment); + self.file_comment_length = @intCast(self.file_comment.len); try writer.writeIntLittle(u32, SIG); try writer.writeIntLittle(u16, self.version_made_by); try writer.writeIntLittle(u16, self.version_needed_to_extract); - try writer.writeIntLittle(u16, self.general_purpose_bit_flag); + try writer.writeIntLittle(u16, @bitCast(self.general_purpose_bit_flag)); try writer.writeIntLittle(u16, @intFromEnum(self.compression_method)); try writer.writeIntLittle(u16, self.last_mod_file_time); try writer.writeIntLittle(u16, self.last_mod_file_date); @@ -251,6 +251,7 @@ const CentralDirectoryHeader = struct { fn from(allocator: std.mem.Allocator, lfh: LocalFileHeader, offset: u32) !CentralDirectoryHeader { // TODO generics return .{ + .version_made_by = 0, .version_needed_to_extract = lfh.version_needed_to_extract, .general_purpose_bit_flag = lfh.general_purpose_bit_flag, .compression_method = lfh.compression_method, @@ -266,9 +267,9 @@ const CentralDirectoryHeader = struct { .internal_file_attributes = 0, .external_file_attributes = 0, .relative_offset_of_local_header = offset, - .file_name = try allocator.dupe(lfh.file_name), - .extra_field = try allocator.dupe(lfh.extra_field), - .file_comment = [_]u8{}, + .file_name = try allocator.dupe(u8, lfh.file_name), + .extra_field = try allocator.dupe(u8, lfh.extra_field), + .file_comment = &[_]u8{}, }; } @@ -350,14 +351,16 @@ const LocalFileHeader = struct { } fn from(allocator: std.mem.Allocator, path: []const u8) !LocalFileHeader { + std.log.err("lfh.from {s}", .{path}); const file = try std.fs.cwd().openFile(path, .{}); defer file.close(); const md = try file.metadata(); var rdr = file.reader(); var cw = std.io.countingWriter(std.io.null_writer); - var comp = try std.compress.deflate.compressor(allocator, cw, .{}); + var comp = try std.compress.deflate.compressor(allocator, cw.writer(), .{}); defer comp.deinit(); const written, const crc32 = try pump_returning(rdr, comp.writer()); + return .{ .version_needed_to_extract = 1, .general_purpose_bit_flag = std.bit_set.IntegerBitSet(16).initEmpty(), @@ -367,9 +370,9 @@ const LocalFileHeader = struct { .crc32 = crc32, .compressed_size = @intCast(written), .uncompressed_size = @intCast(md.size()), // TODO zip64 support - .file_name_length = path.len, + .file_name_length = @intCast(path.len), .extra_field_length = 0, - .file_name = allocator.dupe(u8, path), + .file_name = try allocator.dupe(u8, path), .extra_field = &[_]u8{}, }; } @@ -379,8 +382,8 @@ const LocalFileHeader = struct { self.extra_field_length = @intCast(self.extra_field.len); try writer.writeIntLittle(u32, SIG); try writer.writeIntLittle(u16, self.version_needed_to_extract); - try writer.writeIntLittle(u16, self.general_purpose_bit_flag); - try writer.writeIntLittle(u16, self.compression_method); + try writer.writeIntLittle(u16, @bitCast(self.general_purpose_bit_flag.mask)); + try writer.writeIntLittle(u16, @intFromEnum(self.compression_method)); try writer.writeIntLittle(u16, self.last_mod_file_time); try writer.writeIntLittle(u16, self.last_mod_file_date); try writer.writeIntLittle(u32, self.crc32); @@ -437,6 +440,35 @@ const LocalFileHeader = struct { }, } } + + fn write_data(self: *LocalFileHeader, allocator: std.mem.Allocator, reader: anytype, writer: anytype) !void { + const is_encrypted = self.general_purpose_bit_flag.isSet(0); + if (is_encrypted) return error.EncryptionNotSupported; + + if (self.is_dir()) { + if (self.compressed_size != 0) { + // directories can't have a size, this is very likely wrong. + return error.InvalidFileName; + } + return; // Do nothing here. If we were definitely writing to a filesystem we could make an empty dir I guess. + } + + switch (self.compression_method) { + .store => { + _ = try pump_returning(reader, writer); + }, + .deflate => { + var comp = try std.compress.deflate.compressor(allocator, writer, .{}); + defer comp.deinit(); + var decomp_writer = comp.writer(); + _ = try pump_returning(reader, decomp_writer); + }, + else => { + std.log.err("compression method {} not supported", .{self.compression_method}); + return error.CompressionMethodNotSupported; + }, + } + } }; const Dynamic = struct { @@ -535,43 +567,49 @@ pub const WriteOptions = struct {}; /// Write a whole zip file pub fn write_zip(allocator: std.mem.Allocator, path_iterator: anytype, writer: anytype) !void { var cw = std.io.countingWriter(writer); + var cww = cw.writer(); var zf = @This().empty(allocator); defer zf.deinit(); while (path_iterator.next()) |path| { - const file = try std.fs.cwd().openFile(path, .{}); - defer file.close(); var lfh = try LocalFileHeader.from(allocator, path); defer lfh.deinit(allocator); - var cdh = try CentralDirectoryHeader.from(allocator, lfh, cw.bytes_written); + const file = try std.fs.cwd().openFile(path, .{}); + defer file.close(); + var cdh = try CentralDirectoryHeader.from(allocator, lfh, @intCast(cw.bytes_written)); try zf.central_directory_headers.append(cdh); - try lfh.write(cw); - _ = pump_returning(file.reader(), cw); + std.log.err("lfh {any}", .{lfh}); + try lfh.write(cww); + try lfh.write_data(allocator, file.reader(), cww); } const central_dir_offset = cw.bytes_written; - for (zf.central_directory_headers.items) |cdh| { - try cdh.write(cw); + for (0..zf.central_directory_headers.items.len) |i| { + var cdh = zf.central_directory_headers.items[i]; + std.log.err("cdh {any}", .{cdh}); + try cdh.write(cww); } const size_of_central_dir = cw.bytes_written - central_dir_offset; var eocdr = EndOfCentralDirectoryRecord{ .disk_number_this = 0, .disk_number_central_dir_start = 0, - .total_central_dir_entries_on_this_disk = zf.central_directory_headers.items.len, - .total_central_dir_entries = zf.central_directory_headers.items.len, - .size_of_central_dir = size_of_central_dir, - .central_dir_offset = central_dir_offset, + .total_central_dir_entries_on_this_disk = @intCast(zf.central_directory_headers.items.len), + .total_central_dir_entries = @intCast(zf.central_directory_headers.items.len), + .size_of_central_dir = @intCast(size_of_central_dir), + .central_dir_offset = @intCast(central_dir_offset), // TODO zip64 support .comment_length = 0, .comment = &[_]u8{}, }; - try eocdr.write(cw); + std.log.err("eocdr {any}", .{eocdr}); + try eocdr.write(cww); } const TestFileIter = struct { files: [2][]const u8, ix: usize, - fn next(self: TestFileIter) ?[]const u8 { + fn next(self: *TestFileIter) ?[]const u8 { if (self.ix < 2) { - defer self.ix += 1; - return self.files[self.ix]; + const res = self.files[self.ix]; + self.ix += 1; + return res; } return null; } @@ -579,72 +617,80 @@ const TestFileIter = struct { test "write" { const allocator = std.testing.allocator; - const out = try std.fs.cwd().createFile("test_out.zip", .{}); - defer out.close(); - const tfi = TestFileIter{ - .ix = 0, - .files = [_][]const u8{ "src/foo.txt", "src/bar.txt" }, - }; - try write_zip(allocator, tfi, out.writer()); -} - -test "open stream" { - const test_zip = @embedFile("hello.zip"); - var fbs = std.io.fixedBufferStream(test_zip); - var zf = try @This().from(std.testing.allocator, &fbs); - defer zf.deinit(); - try std.testing.expectEqual(zf.count_files(), 2); - try std.testing.expectEqualStrings(zf.file_name(0), "hello.txt"); - try std.testing.expectEqualStrings(zf.file_name(1), "foo.txt"); -} - -test "open file" { - const test_zip = try std.fs.cwd().openFile("src/hello.zip", .{}); - var zf = try @This().from(std.testing.allocator, &test_zip); - defer zf.deinit(); - try std.testing.expectEqual(zf.count_files(), 2); - try std.testing.expectEqualStrings(zf.file_name(0), "hello.txt"); - try std.testing.expectEqualStrings(zf.file_name(1), "foo.txt"); -} + { + const out = try std.fs.cwd().createFile("test_out.zip", .{}); + defer out.close(); + var tfi = TestFileIter{ + .ix = 0, + .files = [_][]const u8{ "src/foo.txt", "src/bar.txt" }, + }; + try write_zip(allocator, &tfi, out.writer()); + } -test "extract stored" { - const test_zip = @embedFile("hello.zip"); - var fbs = std.io.fixedBufferStream(test_zip); - var zf = try @This().from(std.testing.allocator, &fbs); - defer zf.deinit(); - var out = [_]u8{0} ** 1024; - var fbs_out = std.io.fixedBufferStream(&out); - var writer = fbs_out.writer(); - try zf.extract(0, &fbs, writer); - try std.testing.expectEqualStrings("Hello, Zip!", fbs_out.getWritten()); - fbs_out.reset(); - try zf.extract(1, &fbs, writer); - try std.testing.expectEqualStrings("hi there\n", fbs_out.getWritten()); + { + const out = try std.fs.cwd().openFile("test_out.zip", .{}); + defer out.close(); + try pipeToFileSystem(try std.fs.cwd().makeOpenPath("test_out", .{}), out.reader(), .{.allocator = allocator}); + } } -test "extract deflate" { - const test_zip = @embedFile("deflate.zip"); - var fbs = std.io.fixedBufferStream(test_zip); - var zf = try @This().from(std.testing.allocator, &fbs); - defer zf.deinit(); - var out = [_]u8{0} ** 1024; - var fbs_out = std.io.fixedBufferStream(&out); - var writer = fbs_out.writer(); - try std.testing.expectEqualStrings("Here is a comment :)", zf.file_comment(0)); - try std.testing.expectEqual(@This().CompressionMethod.deflate, zf.central_directory_headers.items[0].compression_method); - try zf.extract(0, &fbs, writer); - try std.testing.expectEqualStrings(@embedFile("foo.txt"), fbs_out.getWritten()); -} -test "subdir" { - const test_zip = @embedFile("subfolder.zip"); - var fbs = std.io.fixedBufferStream(test_zip); - var zf = try @This().from(std.testing.allocator, &fbs); - defer zf.deinit(); - try std.testing.expectEqual(true, zf.is_dir(0)); - try std.testing.expectEqual(false, zf.is_dir(1)); -} -test "pipe to filesystem" { - var f = try std.fs.cwd().openFile("src/subfolder.zip", .{}); - defer f.close(); - try pipeToFileSystem(try std.fs.cwd().makeOpenPath("test", .{}), f.reader(), .{ .allocator = std.testing.allocator }); -} +// test "open stream" { +// const test_zip = @embedFile("hello.zip"); +// var fbs = std.io.fixedBufferStream(test_zip); +// var zf = try @This().from(std.testing.allocator, &fbs); +// defer zf.deinit(); +// try std.testing.expectEqual(zf.count_files(), 2); +// try std.testing.expectEqualStrings(zf.file_name(0), "hello.txt"); +// try std.testing.expectEqualStrings(zf.file_name(1), "foo.txt"); +// } + +// test "open file" { +// const test_zip = try std.fs.cwd().openFile("src/hello.zip", .{}); +// var zf = try @This().from(std.testing.allocator, &test_zip); +// defer zf.deinit(); +// try std.testing.expectEqual(zf.count_files(), 2); +// try std.testing.expectEqualStrings(zf.file_name(0), "hello.txt"); +// try std.testing.expectEqualStrings(zf.file_name(1), "foo.txt"); +// } + +// test "extract stored" { +// const test_zip = @embedFile("hello.zip"); +// var fbs = std.io.fixedBufferStream(test_zip); +// var zf = try @This().from(std.testing.allocator, &fbs); +// defer zf.deinit(); +// var out = [_]u8{0} ** 1024; +// var fbs_out = std.io.fixedBufferStream(&out); +// var writer = fbs_out.writer(); +// try zf.extract(0, &fbs, writer); +// try std.testing.expectEqualStrings("Hello, Zip!", fbs_out.getWritten()); +// fbs_out.reset(); +// try zf.extract(1, &fbs, writer); +// try std.testing.expectEqualStrings("hi there\n", fbs_out.getWritten()); +// } + +// test "extract deflate" { +// const test_zip = @embedFile("deflate.zip"); +// var fbs = std.io.fixedBufferStream(test_zip); +// var zf = try @This().from(std.testing.allocator, &fbs); +// defer zf.deinit(); +// var out = [_]u8{0} ** 1024; +// var fbs_out = std.io.fixedBufferStream(&out); +// var writer = fbs_out.writer(); +// try std.testing.expectEqualStrings("Here is a comment :)", zf.file_comment(0)); +// try std.testing.expectEqual(@This().CompressionMethod.deflate, zf.central_directory_headers.items[0].compression_method); +// try zf.extract(0, &fbs, writer); +// try std.testing.expectEqualStrings(@embedFile("foo.txt"), fbs_out.getWritten()); +// } +// test "subdir" { +// const test_zip = @embedFile("subfolder.zip"); +// var fbs = std.io.fixedBufferStream(test_zip); +// var zf = try @This().from(std.testing.allocator, &fbs); +// defer zf.deinit(); +// try std.testing.expectEqual(true, zf.is_dir(0)); +// try std.testing.expectEqual(false, zf.is_dir(1)); +// } +// test "pipe to filesystem" { +// var f = try std.fs.cwd().openFile("src/subfolder.zip", .{}); +// defer f.close(); +// try pipeToFileSystem(try std.fs.cwd().makeOpenPath("test", .{}), f.reader(), .{ .allocator = std.testing.allocator }); +// } diff --git a/test/foo/foo.txt b/test/foo/foo.txt deleted file mode 100644 index decec2d..0000000 --- a/test/foo/foo.txt +++ /dev/null @@ -1 +0,0 @@ - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. \ No newline at end of file diff --git a/test_out.zip b/test_out.zip new file mode 100644 index 0000000..6e63426 Binary files /dev/null and b/test_out.zip differ diff --git a/test_out/src/bar.txt b/test_out/src/bar.txt new file mode 100644 index 0000000..e69de29 diff --git a/test_out/src/foo.txt b/test_out/src/foo.txt new file mode 100644 index 0000000..e69de29 -- cgit v1.2.3-ZIG