From 3f1c0aa2f8970347372c249651a434091f5401cd Mon Sep 17 00:00:00 2001 From: duck Date: Sat, 30 Aug 2025 23:52:54 +0500 Subject: [PATCH] Async asset loading --- src/assets.zig | 325 ++++++++++++++++++++++++++++++++--------- src/assets/file.zig | 14 ++ src/assets/texture.zig | 34 +++++ src/game.zig | 1 + src/graphics.zig | 4 +- src/world.zig | 9 +- 6 files changed, 309 insertions(+), 78 deletions(-) create mode 100644 src/assets/file.zig create mode 100644 src/assets/texture.zig diff --git a/src/assets.zig b/src/assets.zig index 03a43a3..7816609 100644 --- a/src/assets.zig +++ b/src/assets.zig @@ -1,104 +1,285 @@ const std = @import("std"); -const sdl = @import("sdl"); const err = @import("error.zig"); -const c = @import("c.zig"); -const comp = @import("components.zig"); const Game = @import("game.zig"); -const Graphics = @import("graphics.zig"); +const FileLoader = @import("assets/file.zig"); +const TextureLoader = @import("assets/texture.zig"); const Assets = @This(); -const Storage = comp.Storage(Asset, .{}); +pub const Texture = AssetContainer(TextureLoader); -var storage: Storage = undefined; +const WORKERS_MAX = 4; +var next_worker_update: usize = 0; +var workers: [WORKERS_MAX]WorkerState = undefined; +const WorkerState = struct { + running: bool = false, + thread: ?std.Thread = null, +}; + +const AssetMap = std.HashMapUnmanaged(AssetId, *AssetCell, AssetContext, 80); +var asset_map_mutex: std.Thread.Mutex = .{}; +var asset_map: AssetMap = undefined; + +const RequestBoard = std.ArrayListUnmanaged(*AssetCell); +var request_board_mutex: std.Thread.Mutex = .{}; +var request_board: RequestBoard = undefined; +var request_board_counter: usize = 0; + +const FreeBoard = std.ArrayListUnmanaged(*AssetCell); +var free_board_mutex: std.Thread.Mutex = .{}; +var free_board: FreeBoard = undefined; + +const AssetId = struct { + type: AssetType, + path: []const u8, +}; +const AssetContext = struct { + pub fn hash(self: @This(), key: AssetId) u64 { + _ = self; + var hasher = std.hash.Wyhash.init(@intFromEnum(key.type)); + hasher.update(key.path); + return hasher.final(); + } + pub fn eql(self: @This(), a: AssetId, b: AssetId) bool { + _ = self; + return a.type == b.type and std.mem.eql(u8, a.path, b.path); + } +}; + +pub const LoadError = error{ + DependencyError, +} || std.mem.Allocator.Error || std.fs.File.OpenError || std.fs.File.ReadError; pub const AssetType = enum { + file, texture, -}; -pub const Texture = struct { - handle: Storage.Key, + + pub fn getType(comptime self: @This()) type { + return switch (self) { + .file => FileLoader, + .texture => TextureLoader, + }; + } }; -const Asset = struct { +const AssetState = union(enum) { + not_loaded, + loaded, + fail: LoadError, +}; + +pub const AssetCell = struct { + mutex: std.Thread.Mutex, + type: AssetType, + data: *void, path: []const u8, - data: union(AssetType) { - texture: AssetTexture, - }, -}; -pub const AssetTexture = struct { - texture: *sdl.GPUTexture, - sampler: *sdl.GPUSampler, + loader: *const fn (*AssetCell, std.mem.Allocator) LoadError!void, + unloader: *const fn (*AssetCell, std.mem.Allocator) void, + state: AssetState, + counter: usize, + + fn load(self: *AssetCell, alloc: std.mem.Allocator) void { + self.loader(self, alloc) catch |e| { + self.state = .{ .fail = e }; + return; + }; + self.state = .loaded; + } + fn unload(self: *AssetCell, alloc: std.mem.Allocator) void { + self.unloader(self, alloc); + } }; +pub fn AssetContainer(comptime T: type) type { + return struct { + data_pointer: ?*T = null, + asset_pointer: *AssetCell, + last_state: AssetState = .not_loaded, + + pub fn get(self: *@This()) ?*T { + switch (self.last_state) { + .loaded => { + @branchHint(.likely); + return self.data_pointer; + }, + .fail => { + return null; + }, + .not_loaded => { + if (self.asset_pointer.mutex.tryLock()) { + defer self.asset_pointer.mutex.unlock(); + self.last_state = self.asset_pointer.state; + } + if (self.last_state == .loaded) { + self.data_pointer = @alignCast(@ptrCast(self.asset_pointer.data)); + return self.data_pointer; + } else return null; + }, + } + } + /// To be used by worker threads to request other assets + pub fn getSync(self: *@This()) !*T { + sw: switch (self.last_state) { + .loaded => { + return self.data_pointer.?; + }, + .fail => |e| { + return e; + }, + .not_loaded => { + // TODO: Do something else while the asset is locked? + self.asset_pointer.mutex.lock(); + defer self.asset_pointer.mutex.unlock(); + self.asset_pointer.load(Game.alloc); + self.last_state = self.asset_pointer.state; + if (self.last_state == .loaded) { + self.data_pointer = @alignCast(@ptrCast(self.asset_pointer.data)); + } + continue :sw self.last_state; + }, + } + } + }; +} + pub fn init() void { - Assets.storage = Storage.init(); + Assets.next_worker_update = 0; + Assets.workers = .{WorkerState{}} ** WORKERS_MAX; + Assets.asset_map_mutex = .{}; + Assets.asset_map = AssetMap.empty; + Assets.request_board_mutex = .{}; + Assets.request_board = RequestBoard.empty; + Assets.request_board_counter = 0; + Assets.free_board_mutex = .{}; + Assets.free_board = FreeBoard.empty; } pub fn deinit() void { - var iter = Assets.storage.iter(); + for (&Assets.workers) |*worker| { + if (worker.thread == null) continue; + worker.thread.?.join(); + } + var iter = Assets.asset_map.valueIterator(); while (iter.next()) |asset| { - Assets.freeAsset(asset); + std.debug.assert(asset.*.counter == 0); + if (asset.*.state == .loaded) + asset.*.unload(Game.alloc); + Game.alloc.destroy(asset.*); } - Assets.storage.deinit(); + Assets.asset_map.clearAndFree(Game.alloc); + Assets.request_board.clearAndFree(Game.alloc); + Assets.free_board.clearAndFree(Game.alloc); } -pub fn load(comptime asset_type: AssetType, path: []const u8) typeFromAssetType(asset_type) { - switch (asset_type) { - .texture => { - const data = loadFile(Game.alloc, path) catch |e| err.file(e, path); - var x: i32 = undefined; - var y: i32 = undefined; - var z: i32 = undefined; - const image = c.stbi_load_from_memory(@ptrCast(data), @intCast(data.len), &x, &y, &z, 4); - Game.alloc.free(data); - if (image == null) err.stbi(); - const image_slice = image[0..@intCast(x * y * z)]; - const texture, const sampler = Graphics.loadTexture(@intCast(x), @intCast(y), image_slice); - c.stbi_image_free(image); - return .{ .handle = Assets.storage.add(.{ - .path = path, - .data = .{ .texture = .{ - .texture = texture, - .sampler = sampler, - } }, - }) }; - }, +pub fn update() void { + const worker = &Assets.workers[Assets.next_worker_update]; + if (!@atomicLoad(bool, &worker.running, .acquire) and worker.thread != null) { + worker.thread.?.join(); + worker.thread = null; } -} -pub fn free(asset: anytype) void { - if (Assets.storage.free(asset.handle)) |stored| { - freeAsset(stored); + if (worker.thread == null and @atomicLoad(usize, &Assets.request_board_counter, .monotonic) > 4 * Assets.next_worker_update) { + worker.running = true; + worker.thread = std.Thread.spawn(.{}, loaderLoop, .{Assets.next_worker_update}) catch err.oom(); } -} -pub fn freeAsset(asset: *Asset) void { - switch (asset.data) { - .texture => { - Graphics.unloadTexture(asset.data.texture.texture, asset.data.texture.sampler); - }, + + Assets.next_worker_update += 1; + if (Assets.next_worker_update >= WORKERS_MAX) { + Assets.next_worker_update = 0; } -} -pub fn get(asset: anytype) ?assetTypeFromType(@TypeOf(asset)) { - if (Assets.storage.get(asset.handle)) |stored| { - switch (@TypeOf(asset)) { - Texture => { - return stored.data.texture; - }, - else => @compileError("Cannot get asset of type " ++ @typeName(@TypeOf(asset))), + + Assets.free_board_mutex.lock(); + defer Assets.free_board_mutex.unlock(); + if (Assets.free_board.items.len == 0) return; + + // TODO: Delegate freeing to worker threads? + Assets.asset_map_mutex.lock(); + defer Assets.asset_map_mutex.unlock(); + while (Assets.free_board.pop()) |request| { + if (@atomicLoad(usize, &request.counter, .monotonic) == 0) { + if (!Assets.asset_map.remove(.{ .type = request.type, .path = request.path })) continue; + request.unload(Game.alloc); + Game.alloc.destroy(request); } } - unreachable; } -fn loadFile(alloc: std.mem.Allocator, path: []const u8) ![]u8 { - const file = try std.fs.cwd().openFile(path, .{}); - defer file.close(); - return file.readToEndAlloc(alloc, std.math.maxInt(i32)); +pub fn load(comptime asset_type: AssetType, path: []const u8) AssetContainer(asset_type.getType()) { + const asset = mapAsset(asset_type, path); + { + Assets.request_board_mutex.lock(); + Assets.request_board.append(Game.alloc, asset) catch err.oom(); + _ = @atomicRmw(usize, &Assets.request_board_counter, .Add, 1, .monotonic); + Assets.request_board_mutex.unlock(); + } + return .{ .asset_pointer = asset }; } -fn typeFromAssetType(comptime asset_type: AssetType) type { - return switch (asset_type) { - .texture => Texture, + +pub fn free(asset: anytype) void { + const prev = @atomicRmw(usize, &asset.asset_pointer.counter, .Sub, 1, .monotonic); + if (prev == 1) { + Assets.free_board_mutex.lock(); + Assets.free_board.append(Game.alloc, asset.asset_pointer) catch err.oom(); + Assets.free_board_mutex.unlock(); + } +} + +fn loaderLoop(worker_id: usize) void { + var processed: usize = 0; + defer @atomicStore(bool, &Assets.workers[worker_id].running, false, .release); + while (true) { + const asset = blk: { + Assets.request_board_mutex.lock(); + defer Assets.request_board_mutex.unlock(); + + const request = Assets.request_board.pop() orelse return; + _ = @atomicRmw(usize, &Assets.request_board_counter, .Sub, 1, .monotonic); + break :blk request; + }; + + defer processed += 1; + asset.mutex.lock(); + if (asset.state == .not_loaded) + asset.load(Game.alloc); + asset.mutex.unlock(); + } +} + +fn mapAsset(comptime asset_type: AssetType, path: []const u8) *AssetCell { + Assets.asset_map_mutex.lock(); + defer Assets.asset_map_mutex.unlock(); + + const res = Assets.asset_map.getOrPut(Game.alloc, .{ .type = asset_type, .path = path }) catch err.oom(); + if (!res.found_existing) { + res.value_ptr.* = Game.alloc.create(AssetCell) catch err.oom(); + res.value_ptr.*.* = .{ + .mutex = .{}, + .type = asset_type, + .data = undefined, + .path = path, + .loader = Assets.makeLoader(asset_type.getType(), asset_type.getType().load), + .unloader = Assets.makeUnloader(asset_type.getType(), asset_type.getType().unload), + .state = .not_loaded, + .counter = 1, + }; + } else _ = @atomicRmw(usize, &res.value_ptr.*.counter, .Add, 1, .monotonic); + return res.value_ptr.*; +} + +fn makeLoader(comptime T: type, comptime func: *const fn ([]const u8, std.mem.Allocator) LoadError!T) *const fn (*AssetCell, std.mem.Allocator) LoadError!void { + const Container = struct { + pub fn loader(cell: *AssetCell, alloc: std.mem.Allocator) LoadError!void { + const mem = try alloc.create(T); + errdefer alloc.destroy(mem); + mem.* = try func(cell.path, alloc); + cell.data = @ptrCast(mem); + } }; + return Container.loader; } -fn assetTypeFromType(comptime T: type) type { - return switch (T) { - Texture => AssetTexture, - else => unreachable, + +fn makeUnloader(comptime T: type, comptime func: *const fn (T, std.mem.Allocator) void) *const fn (*AssetCell, std.mem.Allocator) void { + const Container = struct { + pub fn unloader(cell: *AssetCell, alloc: std.mem.Allocator) void { + func(@as(*T, @alignCast(@ptrCast(cell.data))).*, alloc); + alloc.destroy(@as(*T, @alignCast(@ptrCast(cell.data)))); + } }; + return Container.unloader; } diff --git a/src/assets/file.zig b/src/assets/file.zig new file mode 100644 index 0000000..2b7f9eb --- /dev/null +++ b/src/assets/file.zig @@ -0,0 +1,14 @@ +const std = @import("std"); +const Assets = @import("../assets.zig"); + +bytes: []u8, + +pub fn load(path: []const u8, alloc: std.mem.Allocator) Assets.LoadError!@This() { + const file = try std.fs.cwd().openFile(path, .{}); + defer file.close(); + return .{ .bytes = try file.readToEndAlloc(alloc, std.math.maxInt(i32)) }; +} + +pub fn unload(self: @This(), alloc: std.mem.Allocator) void { + alloc.free(self.bytes); +} diff --git a/src/assets/texture.zig b/src/assets/texture.zig new file mode 100644 index 0000000..ecb0959 --- /dev/null +++ b/src/assets/texture.zig @@ -0,0 +1,34 @@ +const std = @import("std"); +const sdl = @import("sdl"); +const err = @import("../error.zig"); +const c = @import("../c.zig"); +const Assets = @import("../assets.zig"); +const Graphics = @import("../graphics.zig"); + +texture: *sdl.GPUTexture, +sampler: *sdl.GPUSampler, + +pub fn load(path: []const u8, alloc: std.mem.Allocator) Assets.LoadError!@This() { + _ = alloc; + var file = Assets.load(.file, path); + defer Assets.free(file); + const data = (try file.getSync()).bytes; + + var x: i32 = undefined; + var y: i32 = undefined; + var z: i32 = undefined; + const image = c.stbi_load_from_memory(@ptrCast(data), @intCast(data.len), &x, &y, &z, 4); + if (image == null) err.stbi(); + const image_slice = image[0..@intCast(x * y * z)]; + const texture, const sampler = Graphics.loadTexture(@intCast(x), @intCast(y), image_slice); + c.stbi_image_free(image); + return .{ + .texture = texture, + .sampler = sampler, + }; +} + +pub fn unload(self: @This(), alloc: std.mem.Allocator) void { + _ = alloc; + Graphics.unloadTexture(self.texture, self.sampler); +} diff --git a/src/game.zig b/src/game.zig index 0b084a0..117bd9b 100644 --- a/src/game.zig +++ b/src/game.zig @@ -62,6 +62,7 @@ pub fn run() void { World.draw(); Game.endDraw(); } + Assets.update(); } } diff --git a/src/graphics.zig b/src/graphics.zig index ca9d93b..14b0f1c 100644 --- a/src/graphics.zig +++ b/src/graphics.zig @@ -420,9 +420,9 @@ pub fn clearDepth() void { sdl.PushGPUVertexUniformData(Graphics.command_buffer, 0, &Graphics.camera.matrix, 16 * 4); } -pub fn drawMesh(mesh: Mesh, texture: Assets.Texture, transform: Transform) void { +pub fn drawMesh(mesh: Mesh, texture: *Assets.Texture, transform: Transform) void { if (Graphics.render_pass == null) return; - const asset_texture = Assets.get(texture) orelse return; + const asset_texture = texture.get() orelse return; sdl.PushGPUVertexUniformData(Graphics.command_buffer, 1, &transform.matrix(), 16 * 4); sdl.BindGPUFragmentSamplers(Graphics.render_pass, 0, &sdl.GPUTextureSamplerBinding{ diff --git a/src/world.zig b/src/world.zig index fd22d53..42e10f0 100644 --- a/src/world.zig +++ b/src/world.zig @@ -91,6 +91,7 @@ pub fn initDebug() void { }; World.object_map.put(Game.alloc, @intCast(i), i) catch err.oom(); } + Assets.free(Assets.load(.texture, "data/yakuza.png")); World.plane_mesh = Graphics.loadMesh(@ptrCast(&PLANE_MESH_DATA)); World.cube_mesh = Graphics.loadMesh(@ptrCast(&CUBE_MESH_DATA)); @@ -323,16 +324,16 @@ pub fn updateObject(object: *Object, delta: f32) void { } pub fn draw() void { - Graphics.drawMesh(World.table_mesh, World.texture, .{}); + Graphics.drawMesh(World.table_mesh, &World.texture, .{}); for (World.objects.items) |*object| { if (object.parent != .dock) - Graphics.drawMesh(object.mesh, object.texture, object.drawingTransform()); + Graphics.drawMesh(object.mesh, &object.texture, object.drawingTransform()); } Graphics.drawMesh( World.plane_mesh, - World.hand_texture, + &World.hand_texture, Graphics.Transform.combineTransforms( .{ .position = .{ World.hand_scale * 0.5, -World.hand_scale * 0.5, 0 }, @@ -345,7 +346,7 @@ pub fn draw() void { Graphics.clearDepth(); for (World.objects.items) |*object| { if (object.parent == .dock) - Graphics.drawMesh(object.mesh, object.texture, object.drawingTransform()); + Graphics.drawMesh(object.mesh, &object.texture, object.drawingTransform()); } }