New modding system

server/game/game_server.ts

@@ -24,7 +24,7 @@ import {
 	PROTOCOL_VERSION,
 	ServerMessage,
 } from "$/common/protocol.ts";
-import type { AtlasListing, ModListing, RecipeBook } from "$/common/mod_loader.ts";
+import type { ModListing, RecipeBook } from "$/common/mod_loader.ts";
 import type { WorldgenSetup } from "$/common/generation.ts";
 import { CYCLE_TICKS, format_clock, time_of_day, TIMES_OF_DAY } from "$/common/time.ts";
 import { ModRuntime, run_guarded } from "./mod_runtime.ts";
@@ -91,7 +91,6 @@ interface SavedTile {
 // the loaded mods, see load_mods.ts
 export interface GameMods {
 	listings: ModListing[];
-	atlas: AtlasListing;
 	recipes: RecipeBook;
 	worldgen?: WorldgenSetup;
 	runtime: ModRuntime;
@@ -485,7 +484,6 @@ export class GameServer {
 						type: "welcome",
 						protocol: PROTOCOL_VERSION,
 						seed: this.world.seed,
-						atlas: this.#mods.atlas,
 						mods: this.#mods.listings,
 					} satisfies ServerMessage,
 				),

server/game/host_protocol.ts

@@ -1,4 +1,4 @@
-import type { AtlasListing, ModData, ModListing } from "$/common/mod_loader.ts";
+import type { ModData, ModListing } from "$/common/mod_loader.ts";
 
 // messages between server/main.ts (the host) and the game server worker
 
@@ -7,7 +7,6 @@ export type HostToGame =
 		type: "init";
 		save: string | undefined;
 		default_seed: string;
-		atlas: AtlasListing;
 		// in load order, with the scripts' code since the worker can't read files
 		mods: { listing: ModListing; data: ModData; server_code?: string; worldgen_code?: string }[];
 	}

server/game/load_mods.ts

@@ -1,7 +1,7 @@
 // starts a game server with its mods: registers their data, then imports worldgen and server scripts.
 // the worker loads built mods, tests can load mods straight from their source folders
 import { register_mod_data } from "$/common/mod_loader.ts";
-import type { AtlasListing, ModData, ModListing } from "$/common/mod_loader.ts";
+import type { ModData, ModListing } from "$/common/mod_loader.ts";
 import { load_worldgen } from "$/common/worldgen_loader.ts";
 import { GameHost, GameServer } from "./game_server.ts";
 import { ModRuntime } from "./mod_runtime.ts";
@@ -18,7 +18,6 @@ export async function start_game(
 	host: GameHost,
 	save: string | undefined,
 	default_seed: string,
-	atlas: AtlasListing,
 	mods: ServerModSource[],
 ): Promise<GameServer> {
 	const recipes = register_mod_data(mods.map((mod) => ({ id: mod.listing.id, data: mod.data })));
@@ -31,7 +30,6 @@ export async function start_game(
 	const runtime = new ModRuntime();
 	const game = new GameServer(host, save, default_seed, {
 		listings: mods.map((mod) => mod.listing),
-		atlas,
 		recipes,
 		worldgen,
 		runtime,

server/game/worker.ts

@@ -54,7 +54,6 @@ async function init(message: Extract<HostToGame, { type: "init" }>) {
 			},
 			message.save,
 			message.default_seed,
-			message.atlas,
 			message.mods.map((mod) => ({
 				listing: mod.listing,
 				data: mod.data,

server/host.ts

@@ -1,20 +1,20 @@
 // the part of a game server that has permissions: files and networking. the game itself runs in a worker with none.
 // server/main.ts runs it as a dedicated server, and the desktop app runs it for singleplayer
 import { serveDir } from "@std/http/file-server";
-import { isAbsolute, join } from "@std/path";
+import { join } from "@std/path";
 import { GameToHost, HostToGame } from "./game/host_protocol.ts";
-import type { ServerModIndex } from "../build.ts";
+import { load_bmods } from "./load_bmods.ts";
 
 const MAX_MESSAGE_SIZE = 4096;
+// in build/, see build.ts
+export const ENGINE_TEXTURES_INDEX = "assets/textures/index.json";
 const SHUTDOWN_TIMEOUT_MS = 5000;
 
 export interface HostOptions {
-	// what deno task build made: the client, assets and mods players download
+	// what deno task build made: the client and the engine's assets
 	build_dir: string;
-	// the server scripts, with index.json
+	// the .bmod files to load
 	server_mods_dir: string;
-	// what the paths in server_mods/index.json are relative to, the folder the build ran in
-	root: string;
 	world_file: string;
 	seed?: string;
 	// the game can't go on, like the worker crashing
@@ -27,9 +27,12 @@ export interface Host {
 	shutdown(): Promise<void>;
 }
 
-export function start_host(options: HostOptions): Host {
-	const mods = read_mods(options);
-	console.log(`Mods: ${mods.mods.map((mod) => `${mod.listing.id} ${mod.listing.version}`).join(", ") || "none"}`);
+// throws when the mods have problems, saying what they are
+export async function start_host(options: HostOptions): Promise<Host> {
+	const mods = await load_bmods(options.server_mods_dir, engine_texture_ids(options.build_dir));
+	console.log(`Mods: ${mods.map((mod) => `${mod.listing.id} ${mod.listing.version}`).join(", ") || "none"}`);
+	// what players download, by path
+	const downloads = new Map(mods.map((mod) => [`/${mod.listing.file}`, mod.client_bytes]));
 
 	const game = new Worker(new URL("./game/worker.ts", import.meta.url), {
 		type: "module",
@@ -78,7 +81,12 @@ export function start_host(options: HostOptions): Host {
 		type: "init",
 		save: read_world(options.world_file),
 		default_seed: options.seed ?? crypto.randomUUID(),
-		...mods,
+		mods: mods.map(({ listing, bmod }) => ({
+			listing,
+			data: bmod.data,
+			server_code: bmod.scripts.server,
+			worldgen_code: bmod.scripts.worldgen,
+		})),
 	});
 
 	function handle_socket(socket: WebSocket) {
@@ -117,6 +125,17 @@ export function start_host(options: HostOptions): Host {
 			if (url.pathname === "/") {
 				return Response.redirect(new URL("/client/", url), 302);
 			}
+			const download = downloads.get(url.pathname);
+			if (download) {
+				// named by its hash, so it never changes and pages on other origins can cache it forever
+				return new Response(download, {
+					headers: {
+						"content-type": "application/zip",
+						"access-control-allow-origin": "*",
+						"cache-control": "public, max-age=31536000, immutable",
+					},
+				});
+			}
 			return serve_static(options.build_dir, req, url);
 		},
 
@@ -155,40 +174,18 @@ function write_world(file: string, data: string) {
 	Deno.renameSync(`${file}.tmp`, file);
 }
 
-// what deno task build made. no index means the build failed or never ran, and the game never runs without its mods
-function read_mods(options: HostOptions): Pick<Extract<HostToGame, { type: "init" }>, "mods" | "atlas"> {
-	let index: ServerModIndex;
-	try {
-		index = JSON.parse(Deno.readTextFileSync(join(options.server_mods_dir, "index.json")));
-	} catch {
-		throw new Error(
-			`No ${options.server_mods_dir}/index.json, run deno task build first (it also reports mod errors)`,
-		);
-	}
-	const from_root = (path: string) => isAbsolute(path) ? path : join(options.root, path);
-	return {
-		atlas: index.atlas,
-		mods: index.mods.map(({ listing, server }) => ({
-			listing,
-			data: JSON.parse(Deno.readTextFileSync(join(options.build_dir, listing.data))),
-			server_code: server ? Deno.readTextFileSync(from_root(server)) : undefined,
-			worldgen_code: listing.worldgen
-				? Deno.readTextFileSync(join(options.build_dir, listing.worldgen))
-				: undefined,
-		})),
-	};
+// the engine's textures, listed by the build next to the client's assets. engine:missing is drawn by code
+export function engine_texture_ids(build_dir: string): string[] {
+	const names: string[] = JSON.parse(Deno.readTextFileSync(join(build_dir, ENGINE_TEXTURES_INDEX)));
+	return ["engine:missing", ...names.map((name) => `engine:${name}`)];
 }
 
-// mod files and the atlas are named by their hash, so they never change and pages on other origins can load them
+// the client and the engine's assets. pages on other origins load the engine's textures from here too
 export async function serve_static(build_dir: string, req: Request, url: URL) {
 	const response = await serveDir(req, { fsRoot: build_dir, quiet: true });
-	const shared = url.pathname.startsWith("/mods/") || url.pathname.startsWith("/assets/");
-	if (shared && response.ok) {
+	if (url.pathname.startsWith("/assets/") && response.ok) {
 		const headers = new Headers(response.headers);
 		headers.set("Access-Control-Allow-Origin", "*");
-		if (url.pathname.startsWith("/mods/") || /^\/assets\/sprites\/textures\.[0-9a-f]+\./.test(url.pathname)) {
-			headers.set("Cache-Control", "public, max-age=31536000, immutable");
-		}
 		return new Response(response.body, { status: response.status, statusText: response.statusText, headers });
 	}
 	return response;

server/load_bmods.ts

@@ -0,0 +1,111 @@
+// the server's mods: every .bmod in server_mods/, checked, sorted so dependencies load first, and with the copy
+// players download (the same mod without its server script) ready to serve
+import { join } from "@std/path";
+import { type Bmod, BMOD_EXTENSION, client_copy, read_bmod, write_bmod } from "$/common/bmod.ts";
+import { check_references, load_order, type LoadedMod } from "$/common/mod_check.ts";
+import type { ModListing } from "$/common/mod_loader.ts";
+
+export interface ServerMod {
+	bmod: Bmod;
+	listing: ModListing;
+	// what's served at listing.file
+	client_bytes: Uint8Array<ArrayBuffer>;
+}
+
+export class ModsError extends Error {
+	constructor(problems: string[]) {
+		super(`Mods have errors:\n${problems.map((p) => `  ${p}`).join("\n")}`);
+		this.name = "ModsError";
+	}
+}
+
+// engine_textures are the engine: texture ids, for checking the textures mods use. a mod with problems stops the
+// whole server from starting, the game never runs with some mods missing
+export async function load_bmods(dir: string, engine_textures: Iterable<string>): Promise<ServerMod[]> {
+	let files: string[];
+	try {
+		files = [...Deno.readDirSync(dir)]
+			.filter((entry) => entry.isFile && entry.name.endsWith(BMOD_EXTENSION))
+			.map((entry) => entry.name)
+			.sort();
+	} catch (e) {
+		if (e instanceof Deno.errors.NotFound) return [];
+		throw e;
+	}
+
+	const problems: string[] = [];
+	const by_id = new Map<string, { file: string; bmod: Bmod }>();
+	for (const file of files) {
+		try {
+			const bmod = read_bmod(Deno.readFileSync(join(dir, file)), file);
+			const other = by_id.get(bmod.manifest.id);
+			if (other) {
+				problems.push(`${file} and ${other.file} are both the mod "${bmod.manifest.id}"`);
+				continue;
+			}
+			by_id.set(bmod.manifest.id, { file, bmod });
+		} catch (e) {
+			problems.push((e as Error).message);
+		}
+	}
+
+	// the same checks check-mods runs on mod folders
+	const loaded = [...by_id.values()].map(({ file, bmod }) => as_loaded_mod(file, bmod));
+	check_references(loaded, engine_textures);
+	for (const mod of loaded) {
+		const dependencies = (mod.manifest?.dependencies ?? []) as { id: string }[];
+		for (const { id } of dependencies) {
+			if (!by_id.has(id)) mod.report.errors.push(`depends on "${id}", which isn't in ${dir}`);
+		}
+		problems.push(...mod.report.errors.map((error) => `${mod.dir}: ${error}`));
+		for (const warning of mod.report.warnings) console.warn(`${mod.dir}: ${warning}`);
+	}
+	let ordered: LoadedMod[] = [];
+	try {
+		ordered = load_order(loaded);
+	} catch (e) {
+		problems.push((e as Error).message);
+	}
+	if (problems.length > 0) throw new ModsError(problems);
+
+	return await Promise.all(ordered.map(async (mod) => {
+		const bmod = by_id.get(mod.id)!.bmod;
+		const client_bytes = write_bmod(client_copy(bmod));
+		const sha256 = await hex_sha256(client_bytes);
+		return {
+			bmod,
+			client_bytes,
+			listing: {
+				id: bmod.manifest.id,
+				name: bmod.manifest.name,
+				version: bmod.manifest.version,
+				sha256,
+				file: `mods/${sha256}${BMOD_EXTENSION}`,
+				size: client_bytes.length,
+			},
+		};
+	}));
+}
+
+// in the shape the checks in common/mod_check.ts take
+function as_loaded_mod(file: string, bmod: Bmod): LoadedMod {
+	const entries = <T>(list: T[]) => list.map((json) => ({ file: `${file} data.json`, json }));
+	return {
+		id: bmod.manifest.id,
+		dir: file,
+		report: { id: bmod.manifest.id, errors: [], warnings: [] },
+		manifest: bmod.manifest as unknown as Record<string, unknown>,
+		blocks: entries(bmod.data.blocks),
+		models: entries(bmod.data.models),
+		items: entries(bmod.data.items),
+		recipes: entries(bmod.data.recipes),
+		ores: entries(bmod.data.ores),
+		textures: [...bmod.textures.keys()],
+		texture_files: new Map(),
+	};
+}
+
+async function hex_sha256(bytes: Uint8Array<ArrayBuffer>) {
+	const digest = await crypto.subtle.digest("SHA-256", bytes);
+	return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join("");
+}

server/main.ts

@@ -5,10 +5,9 @@ const PORT = Number(Deno.env.get("PORT") ?? 8000);
 
 let host: Host;
 try {
-	host = start_host({
+	host = await start_host({
 		build_dir: Deno.env.get("BUILD_DIR") ?? "build",
 		server_mods_dir: Deno.env.get("SERVER_MODS_DIR") ?? "server_mods",
-		root: ".",
 		world_file: Deno.env.get("WORLD_FILE") ?? "world.json",
 		seed: Deno.env.get("SEED"),
 		on_fatal(message) {