From c5c11a5a40a6058b0b76965d265c9a7d1ccff9f4 Mon Sep 17 00:00:00 2001 From: MysterD Date: Tue, 12 Apr 2022 19:25:06 -0700 Subject: [PATCH] More work on mod hashing/caching --- developer/network.sh | 12 +- src/pc/mods/mod.c | 6 +- src/pc/mods/mod_cache.c | 12 +- src/pc/mods/mod_cache.h | 3 +- src/pc/mods/mods.c | 2 +- src/pc/mods/mods_utils.c | 14 -- src/pc/mods/mods_utils.h | 1 - src/pc/network/network.c | 3 + src/pc/network/packets/packet.c | 3 + src/pc/network/packets/packet.h | 6 + src/pc/network/packets/packet_download.c | 2 +- src/pc/network/packets/packet_mod_list.c | 223 ++++++++++++++++------- src/pc/network/packets/packet_reliable.c | 3 + 13 files changed, 195 insertions(+), 95 deletions(-) diff --git a/developer/network.sh b/developer/network.sh index a2b475a0..0cd2d88f 100755 --- a/developer/network.sh +++ b/developer/network.sh @@ -16,9 +16,9 @@ if [ ! -f "$FILE" ]; then fi # no debug, direct -$FILE --server 27015 --configfile sm64config_server.txt & +$FILE --server 7777 --configfile sm64config_server.txt & sleep 2 -$FILE --client 127.0.0.1 27015 --configfile sm64config_client.txt & +$FILE --client 127.0.0.1 7777 --configfile sm64config_client.txt & exit # no debug, discord @@ -27,11 +27,11 @@ exit #exit # debug on server -#$FILE --client 127.0.0.1 27015 --configfile sm64config_client.txt & > /dev/null -#$WINPTY cgdb $FILE -ex 'break debug_breakpoint_here' -ex 'run --server 27015 --configfile sm64config_server.txt' -ex 'quit' +#$FILE --client 127.0.0.1 7777 --configfile sm64config_client.txt & > /dev/null +#$WINPTY cgdb $FILE -ex 'break debug_breakpoint_here' -ex 'run --server 7777 --configfile sm64config_server.txt' -ex 'quit' #exit # debug on client -$FILE --server 27015 --configfile sm64config_server.txt & > /dev/null -$WINPTY cgdb $FILE -ex 'network_receive_download' -ex 'break debug_breakpoint_here' -ex 'run --client 127.0.0.1 27015 --configfile sm64config_client.txt' -ex 'quit' +$FILE --server 7777 --configfile sm64config_server.txt & > /dev/null +$WINPTY cgdb $FILE -ex 'network_receive_download' -ex 'break debug_breakpoint_here' -ex 'run --client 127.0.0.1 7777 --configfile sm64config_client.txt' -ex 'quit' exit diff --git a/src/pc/mods/mod.c b/src/pc/mods/mod.c index bc2e3649..df50f261 100644 --- a/src/pc/mods/mod.c +++ b/src/pc/mods/mod.c @@ -525,7 +525,7 @@ bool mod_load(struct Mods* mods, char* basePath, char* modName) { struct ModCacheEntry* cache = mod_cache_get_from_path(fullPath); if (cache == NULL) { mod_md5_hash(mod); - mod_cache_add(mod->dataHash, strdup(fullPath)); + mod_cache_add(mod->dataHash, 0, strdup(fullPath)); } return true; @@ -572,13 +572,13 @@ void mod_md5_hash(struct Mod* mod) { mod->hashProcessed = true; if (mod->isDirectory) { - mod_cache_add(mod->dataHash, strdup(mod->basePath)); + mod_cache_add(mod->dataHash, 0, strdup(mod->basePath)); } else { if (!concat_path(path, mod->basePath, mod->files[0].relativePath)) { LOG_ERROR("Failed to combine path for mod hashing."); return; } - mod_cache_add(mod->dataHash, strdup(path)); + mod_cache_add(mod->dataHash, 0, strdup(path)); } } diff --git a/src/pc/mods/mod_cache.c b/src/pc/mods/mod_cache.c index ed34e373..414fb812 100644 --- a/src/pc/mods/mod_cache.c +++ b/src/pc/mods/mod_cache.c @@ -5,7 +5,7 @@ #include "pc/debuglog.h" #define MOD_CACHE_FILENAME "mod.cache" -#define MOD_CACHE_VERSION 1 +#define MOD_CACHE_VERSION 2 struct ModCacheEntry* sModCacheHead = NULL; @@ -49,13 +49,15 @@ struct ModCacheEntry* mod_cache_get_from_path(const char* path) { return NULL; } -void mod_cache_add(u8* dataHash, const char* path) { +void mod_cache_add(u8* dataHash, u64 lastLoaded, const char* path) { if (mod_cache_get_from_hash(dataHash)) { return; } struct ModCacheEntry* node = calloc(1, sizeof(struct ModCacheEntry)); memcpy(node->dataHash, dataHash, 16); + if (lastLoaded == 0) { lastLoaded = clock(); } + node->lastLoaded = lastLoaded; node->path = (char*)path; node->next = NULL; @@ -107,17 +109,20 @@ void mod_cache_load(void) { while (true) { u8 dataHash[16] = { 0 }; + u64 lastLoaded = 0; u16 pathLen; if (fread(dataHash, sizeof(u8), 16, fp) == 0) { break; } + + fread(&lastLoaded, sizeof(u64), 1, fp); fread(&pathLen, sizeof(u16), 1, fp); const char* path = calloc(pathLen + 1, sizeof(u8)); fread((char*)path, sizeof(u8), pathLen + 1, fp); - mod_cache_add(dataHash, path); + mod_cache_add(dataHash, lastLoaded, path); } LOG_INFO("Loading mod cache complete"); @@ -138,6 +143,7 @@ void mod_cache_save(void) { struct ModCacheEntry* node = sModCacheHead; while (node != NULL) { fwrite(node->dataHash, sizeof(u8), 16, fp); + fwrite(node->lastLoaded, sizeof(u64), 1, fp); u16 pathLen = strlen(node->path); fwrite(&pathLen, sizeof(u16), 1, fp); fwrite(node->path, sizeof(u8), pathLen + 1, fp); diff --git a/src/pc/mods/mod_cache.h b/src/pc/mods/mod_cache.h index f45c5650..3b9c56b6 100644 --- a/src/pc/mods/mod_cache.h +++ b/src/pc/mods/mod_cache.h @@ -4,6 +4,7 @@ struct ModCacheEntry { u8 dataHash[16]; + u64 lastLoaded; char* path; struct ModCacheEntry* next; }; @@ -11,7 +12,7 @@ struct ModCacheEntry { void mod_cache_shutdown(void); struct ModCacheEntry* mod_cache_get_from_hash(u8* dataHash); struct ModCacheEntry* mod_cache_get_from_path(const char* path); -void mod_cache_add(u8* dataHash, const char* path); +void mod_cache_add(u8* dataHash, u64 lastLoaded, const char* path); void mod_cache_load(void); void mod_cache_save(void); diff --git a/src/pc/mods/mods.c b/src/pc/mods/mods.c index a4389c94..9baaa465 100644 --- a/src/pc/mods/mods.c +++ b/src/pc/mods/mods.c @@ -62,6 +62,7 @@ void mods_activate(struct Mods* mods) { } // copy enabled entries + gActiveMods.entryCount = 0; gActiveMods.size = 0; for (int i = 0; i < mods->entryCount; i++) { struct Mod* mod = mods->entries[i]; @@ -213,5 +214,4 @@ void mods_shutdown(void) { mods_clear(&gRemoteMods); mods_clear(&gActiveMods); mods_clear(&gLocalMods); - mods_delete_tmp(); } diff --git a/src/pc/mods/mods_utils.c b/src/pc/mods/mods_utils.c index 95003294..500b8767 100644 --- a/src/pc/mods/mods_utils.c +++ b/src/pc/mods/mods_utils.c @@ -94,20 +94,6 @@ static void mods_delete_folder(char* path) { rmdir(path); } -void mods_delete_tmp(void) { - // ensure tmpPath exists - char tmpPath[SYS_MAX_PATH] = { 0 }; - if (snprintf(tmpPath, SYS_MAX_PATH - 1, "%s", fs_get_write_path(TMP_DIRECTORY)) < 0) { - LOG_ERROR("Failed to concat tmp path"); - return; - } - - // sanity - if (strlen(tmpPath) < 1) { return; } - - // delete - mods_delete_folder(tmpPath); -} ////////////////////////////////////////////////////////////////////////////////////////// bool mod_file_full_path(char* destination, struct Mod* mod, struct ModFile* modFile) { diff --git a/src/pc/mods/mods_utils.h b/src/pc/mods/mods_utils.h index 35eb88fa..82549f98 100644 --- a/src/pc/mods/mods_utils.h +++ b/src/pc/mods/mods_utils.h @@ -7,7 +7,6 @@ void mods_size_enforce(struct Mods* mods); void mods_update_selectable(void); -void mods_delete_tmp(void); bool mod_file_full_path(char* destination, struct Mod* mod, struct ModFile* modFile); bool mod_file_create_directories(struct Mod* mod, struct ModFile* modFile); diff --git a/src/pc/network/network.c b/src/pc/network/network.c index 12005898..0e145730 100644 --- a/src/pc/network/network.c +++ b/src/pc/network/network.c @@ -176,6 +176,9 @@ bool network_allow_unknown_local_index(enum PacketType packetType) { || (packetType == PACKET_ACK) || (packetType == PACKET_MOD_LIST_REQUEST) || (packetType == PACKET_MOD_LIST) + || (packetType == PACKET_MOD_LIST_ENTRY) + || (packetType == PACKET_MOD_LIST_FILE) + || (packetType == PACKET_MOD_LIST_DONE) || (packetType == PACKET_DOWNLOAD_REQUEST) || (packetType == PACKET_DOWNLOAD) || (packetType == PACKET_KEEP_ALIVE) diff --git a/src/pc/network/packets/packet.c b/src/pc/network/packets/packet.c index eb1b6389..84b1efc8 100644 --- a/src/pc/network/packets/packet.c +++ b/src/pc/network/packets/packet.c @@ -83,6 +83,9 @@ void packet_process(struct Packet* p) { case PACKET_MOD_LIST: network_receive_mod_list(p); break; case PACKET_DOWNLOAD_REQUEST: network_receive_download_request(p); break; case PACKET_DOWNLOAD: network_receive_download(p); break; + case PACKET_MOD_LIST_ENTRY: network_receive_mod_list_entry(p); break; + case PACKET_MOD_LIST_FILE: network_receive_mod_list_file(p); break; + case PACKET_MOD_LIST_DONE: network_receive_mod_list_done(p); break; case PACKET_LUA_SYNC_TABLE_REQUEST: network_receive_lua_sync_table_request(p); break; case PACKET_LUA_SYNC_TABLE: network_receive_lua_sync_table(p); break; diff --git a/src/pc/network/packets/packet.h b/src/pc/network/packets/packet.h index eaecc313..a8097b5b 100644 --- a/src/pc/network/packets/packet.h +++ b/src/pc/network/packets/packet.h @@ -58,6 +58,9 @@ enum PacketType { PACKET_MOD_LIST, PACKET_DOWNLOAD_REQUEST, PACKET_DOWNLOAD, + PACKET_MOD_LIST_ENTRY, + PACKET_MOD_LIST_FILE, + PACKET_MOD_LIST_DONE, PACKET_LUA_SYNC_TABLE_REQUEST, PACKET_LUA_SYNC_TABLE, @@ -327,6 +330,9 @@ void network_send_mod_list_request(void); void network_receive_mod_list_request(UNUSED struct Packet* p); void network_send_mod_list(void); void network_receive_mod_list(struct Packet* p); +void network_receive_mod_list_entry(struct Packet* p); +void network_receive_mod_list_file(struct Packet* p); +void network_receive_mod_list_done(struct Packet* p); // packet_download.c void network_start_download_requests(void); diff --git a/src/pc/network/packets/packet_download.c b/src/pc/network/packets/packet_download.c index 6a489d93..403907f0 100644 --- a/src/pc/network/packets/packet_download.c +++ b/src/pc/network/packets/packet_download.c @@ -61,7 +61,7 @@ static void mark_groups_loaded_from_hash(void) { if (mod->loadedFromCache) { // if we loaded from cache, mark bytes as downloaded sTotalDownloadBytes += mod->size; - LOG_INFO("Loaded from cache: %s, %lu", mod->name, mod->size); + LOG_INFO("Loaded from cache: %s, %llu", mod->name, mod->size); } else { // if we haven't loaded from cache, we need this offset group u64 ogIndexStart = fileStartOffset / GROUP_SIZE; diff --git a/src/pc/network/packets/packet_mod_list.c b/src/pc/network/packets/packet_mod_list.c index f917a730..7e07f41c 100644 --- a/src/pc/network/packets/packet_mod_list.c +++ b/src/pc/network/packets/packet_mod_list.c @@ -32,6 +32,8 @@ void network_receive_mod_list_request(UNUSED struct Packet* p) { void network_send_mod_list(void) { SOFT_ASSERT(gNetworkType == NT_SERVER); + packet_ordered_begin(); + struct Packet p = { 0 }; packet_init(&p, PACKET_MOD_LIST, true, PLMT_NONE); @@ -40,6 +42,7 @@ void network_send_mod_list(void) { LOG_INFO("sending version: %s", version); packet_write(&p, &version, sizeof(u8) * MAX_VERSION_LENGTH); packet_write(&p, &gActiveMods.entryCount, sizeof(u16)); + network_send_to(0, &p); LOG_INFO("sent mod list (%u):", gActiveMods.entryCount); for (u16 i = 0; i < gActiveMods.entryCount; i++) { @@ -51,6 +54,9 @@ void network_send_mod_list(void) { u16 relativePathLength = strlen(mod->relativePath); u64 modSize = mod->size; + struct Packet p = { 0 }; + packet_init(&p, PACKET_MOD_LIST_ENTRY, true, PLMT_NONE); + packet_write(&p, &i, sizeof(u16)); packet_write(&p, &nameLength, sizeof(u16)); packet_write(&p, mod->name, sizeof(u8) * nameLength); packet_write(&p, &relativePathLength, sizeof(u16)); @@ -58,20 +64,32 @@ void network_send_mod_list(void) { packet_write(&p, &modSize, sizeof(u64)); packet_write(&p, &mod->isDirectory, sizeof(u8)); packet_write(&p, &mod->dataHash[0], sizeof(u8) * 16); + packet_write(&p, &mod->fileCount, sizeof(u16)); + network_send_to(0, &p); LOG_INFO(" '%s': %llu", mod->name, (u64)mod->size); - packet_write(&p, &mod->fileCount, sizeof(u16)); for (u16 j = 0; j < mod->fileCount; j++) { + struct Packet p = { 0 }; + packet_init(&p, PACKET_MOD_LIST_FILE, true, PLMT_NONE); struct ModFile* file = &mod->files[j]; u16 relativePathLength = strlen(file->relativePath); u64 fileSize = file->size; + packet_write(&p, &i, sizeof(u16)); + packet_write(&p, &j, sizeof(u16)); packet_write(&p, &relativePathLength, sizeof(u16)); packet_write(&p, file->relativePath, sizeof(u8) * relativePathLength); packet_write(&p, &fileSize, sizeof(u64)); + network_send_to(0, &p); LOG_INFO(" '%s': %llu", file->relativePath, (u64)file->size); } } - network_send_to(0, &p); + + struct Packet p2 = { 0 }; + packet_init(&p2, PACKET_MOD_LIST_DONE, true, PLMT_NONE); + network_send_to(0, &p2); + + packet_ordered_end(); + } void network_receive_mod_list(struct Packet* p) { @@ -79,7 +97,7 @@ void network_receive_mod_list(struct Packet* p) { if (p->localIndex != UNKNOWN_LOCAL_INDEX) { if (gNetworkPlayerServer == NULL || gNetworkPlayerServer->localIndex != p->localIndex) { - LOG_ERROR("Received download from known local index '%d'", p->localIndex); + LOG_ERROR("Received mod list from known local index '%d'", p->localIndex); return; } } @@ -118,72 +136,147 @@ void network_receive_mod_list(struct Packet* p) { } LOG_INFO("received mod list (%u):", gRemoteMods.entryCount); +} + +void network_receive_mod_list_entry(struct Packet* p) { + SOFT_ASSERT(gNetworkType == NT_CLIENT); + + // make sure it was sent by the server + if (p->localIndex != UNKNOWN_LOCAL_INDEX) { + if (gNetworkPlayerServer == NULL || gNetworkPlayerServer->localIndex != p->localIndex) { + LOG_ERROR("Received download from known local index '%d'", p->localIndex); + return; + } + } + + // get mod index + u16 modIndex = 0; + packet_read(p, &modIndex, sizeof(u16)); + if (modIndex >= gRemoteMods.entryCount) { + LOG_ERROR("Received mod outside of known range"); + return; + } + + // allocate mod entry + gRemoteMods.entries[modIndex] = calloc(1, sizeof(struct Mod)); + struct Mod* mod = gRemoteMods.entries[modIndex]; + if (mod == NULL) { + LOG_ERROR("Failed to allocate remote mod!"); + return; + } + + // get name length + u16 nameLength = 0; + packet_read(p, &nameLength, sizeof(u16)); + if (nameLength > 31) { + LOG_ERROR("Received name with invalid length!"); + return; + } + + // get name + char name[32] = { 0 }; + packet_read(p, name, nameLength * sizeof(u8)); + mod->name = strdup(name); + + // get other fields + u16 relativePathLength = 0; + packet_read(p, &relativePathLength, sizeof(u16)); + packet_read(p, mod->relativePath, relativePathLength * sizeof(u8)); + packet_read(p, &mod->size, sizeof(u64)); + packet_read(p, &mod->isDirectory, sizeof(u8)); + packet_read(p, &mod->dataHash, sizeof(u8) * 16); + normalize_path(mod->relativePath); + LOG_INFO(" '%s': %llu", mod->name, (u64)mod->size); + + // figure out base path + if (mod->isDirectory) { + if (snprintf(mod->basePath, SYS_MAX_PATH - 1, "%s/%s", gRemoteModsBasePath, mod->relativePath) < 0) { + LOG_ERROR("Failed save remote base path!"); + return; + } + normalize_path(mod->basePath); + } else { + if (snprintf(mod->basePath, SYS_MAX_PATH - 1, "%s", gRemoteModsBasePath) < 0) { + LOG_ERROR("Failed save remote base path!"); + return; + } + } + + // sanity check mod size + if (mod->size >= MAX_MOD_SIZE) { + djui_popup_create("Server had too large of a mod.\nQuitting.", 4); + network_shutdown(false); + return; + } + + // get file count and allocate them + packet_read(p, &mod->fileCount, sizeof(u16)); + mod->files = calloc(mod->fileCount, sizeof(struct ModFile)); + if (mod->files == NULL) { + LOG_ERROR("Failed to allocate mod files!"); + return; + } +} + +void network_receive_mod_list_file(struct Packet* p) { + SOFT_ASSERT(gNetworkType == NT_CLIENT); + + if (p->localIndex != UNKNOWN_LOCAL_INDEX) { + if (gNetworkPlayerServer == NULL || gNetworkPlayerServer->localIndex != p->localIndex) { + LOG_ERROR("Received download from known local index '%d'", p->localIndex); + return; + } + } + + // get mod index + u16 modIndex = 0; + packet_read(p, &modIndex, sizeof(u16)); + if (modIndex >= gRemoteMods.entryCount) { + LOG_ERROR("Received mod outside of known range"); + return; + } + struct Mod* mod = gRemoteMods.entries[modIndex]; + if (mod == NULL) { + LOG_ERROR("Received mod file for null mod"); + return; + } + + // get file index + u16 fileIndex = 0; + packet_read(p, &fileIndex, sizeof(u16)); + if (fileIndex >= mod->fileCount) { + LOG_ERROR("Received mod file outside of known range"); + return; + } + struct ModFile* file = &mod->files[fileIndex]; + if (mod == NULL) { + LOG_ERROR("Received null mod file"); + return; + } + + u16 relativePathLength = 0; + packet_read(p, &relativePathLength, sizeof(u16)); + packet_read(p, file->relativePath, relativePathLength * sizeof(u8)); + packet_read(p, &file->size, sizeof(u64)); + file->fp = NULL; + LOG_INFO(" '%s': %llu", file->relativePath, (u64)file->size); + +} + +void network_receive_mod_list_done(struct Packet* p) { + SOFT_ASSERT(gNetworkType == NT_CLIENT); + + if (p->localIndex != UNKNOWN_LOCAL_INDEX) { + if (gNetworkPlayerServer == NULL || gNetworkPlayerServer->localIndex != p->localIndex) { + LOG_ERROR("Received download from known local index '%d'", p->localIndex); + return; + } + } + size_t totalSize = 0; for (u16 i = 0; i < gRemoteMods.entryCount; i++) { - gRemoteMods.entries[i] = calloc(1, sizeof(struct Mod)); struct Mod* mod = gRemoteMods.entries[i]; - if (mod == NULL) { - LOG_ERROR("Failed to allocate remote mod!"); - return; - } - - char name[32] = { 0 }; - u16 nameLength = 0; - - packet_read(p, &nameLength, sizeof(u16)); - if (nameLength > 31) { - LOG_ERROR("Received name with invalid length!"); - return; - } - packet_read(p, name, nameLength * sizeof(u8)); - mod->name = strdup(name); - - u16 relativePathLength = 0; - packet_read(p, &relativePathLength, sizeof(u16)); - packet_read(p, mod->relativePath, relativePathLength * sizeof(u8)); - packet_read(p, &mod->size, sizeof(u64)); - packet_read(p, &mod->isDirectory, sizeof(u8)); - packet_read(p, &mod->dataHash, sizeof(u8) * 16); - normalize_path(mod->relativePath); totalSize += mod->size; - LOG_INFO(" '%s': %llu", mod->name, (u64)mod->size); - - if (mod->isDirectory) { - if (snprintf(mod->basePath, SYS_MAX_PATH - 1, "%s/%s", gRemoteModsBasePath, mod->relativePath) < 0) { - LOG_ERROR("Failed save remote base path!"); - return; - } - normalize_path(mod->basePath); - } else { - if (snprintf(mod->basePath, SYS_MAX_PATH - 1, "%s", gRemoteModsBasePath) < 0) { - LOG_ERROR("Failed save remote base path!"); - return; - } - } - - if (mod->size >= MAX_MOD_SIZE) { - djui_popup_create("Server had too large of a mod.\nQuitting.", 4); - network_shutdown(false); - return; - } - - packet_read(p, &mod->fileCount, sizeof(u16)); - mod->files = calloc(mod->fileCount, sizeof(struct ModFile)); - if (mod->files == NULL) { - LOG_ERROR("Failed to allocate mod files!"); - return; - } - - for (u16 j = 0; j < mod->fileCount; j++) { - struct ModFile* file = &mod->files[j]; - u16 relativePathLength = 0; - packet_read(p, &relativePathLength, sizeof(u16)); - packet_read(p, file->relativePath, relativePathLength * sizeof(u8)); - packet_read(p, &file->size, sizeof(u64)); - file->fp = NULL; - LOG_INFO(" '%s': %llu", file->relativePath, (u64)file->size); - } - mod_load_from_cache(mod); } gRemoteMods.size = totalSize; diff --git a/src/pc/network/packets/packet_reliable.c b/src/pc/network/packets/packet_reliable.c index 3a040b78..aa1262d2 100644 --- a/src/pc/network/packets/packet_reliable.c +++ b/src/pc/network/packets/packet_reliable.c @@ -125,6 +125,9 @@ static float network_adjust_max_elapsed(enum PacketType packetType, float maxEla case PACKET_DOWNLOAD: case PACKET_MOD_LIST_REQUEST: case PACKET_MOD_LIST: + case PACKET_MOD_LIST_ENTRY: + case PACKET_MOD_LIST_FILE: + case PACKET_MOD_LIST_DONE: return 0.2f + maxElapsed * 2.0f; default: return maxElapsed;