Hri7566
/
sm64coopdx
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

More work on mod hashing/caching

This commit is contained in:
MysterD 2022-04-12 19:25:06 -07:00
parent 97a9360529
commit c5c11a5a40
13 changed files with 195 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
developer/network.sh
View File

@ -16,9 +16,9 @@ if [ ! -f "$FILE" ]; then
fi
# no debug, direct
$FILE --server 27015 --configfile sm64config_server.txt &
$FILE --server 7777 --configfile sm64config_server.txt &
sleep 2
$FILE --client 127.0.0.1 27015 --configfile sm64config_client.txt &
$FILE --client 127.0.0.1 7777 --configfile sm64config_client.txt &
exit
# no debug, discord
@ -27,11 +27,11 @@ exit
#exit
# debug on server
#$FILE --client 127.0.0.1 27015 --configfile sm64config_client.txt & > /dev/null
#$WINPTY cgdb $FILE -ex 'break debug_breakpoint_here' -ex 'run --server 27015 --configfile sm64config_server.txt' -ex 'quit'
#$FILE --client 127.0.0.1 7777 --configfile sm64config_client.txt & > /dev/null
#$WINPTY cgdb $FILE -ex 'break debug_breakpoint_here' -ex 'run --server 7777 --configfile sm64config_server.txt' -ex 'quit'
#exit
# debug on client
$FILE --server 27015 --configfile sm64config_server.txt & > /dev/null
$WINPTY cgdb $FILE -ex 'network_receive_download' -ex 'break debug_breakpoint_here' -ex 'run --client 127.0.0.1 27015 --configfile sm64config_client.txt' -ex 'quit'
$FILE --server 7777 --configfile sm64config_server.txt & > /dev/null
$WINPTY cgdb $FILE -ex 'network_receive_download' -ex 'break debug_breakpoint_here' -ex 'run --client 127.0.0.1 7777 --configfile sm64config_client.txt' -ex 'quit'
exit

6
src/pc/mods/mod.c
View File

@ -525,7 +525,7 @@ bool mod_load(struct Mods* mods, char* basePath, char* modName) {
struct ModCacheEntry* cache = mod_cache_get_from_path(fullPath);
if (cache == NULL) {
mod_md5_hash(mod);
mod_cache_add(mod->dataHash, strdup(fullPath));
mod_cache_add(mod->dataHash, 0, strdup(fullPath));
}
return true;
@ -572,13 +572,13 @@ void mod_md5_hash(struct Mod* mod) {
mod->hashProcessed = true;
if (mod->isDirectory) {
mod_cache_add(mod->dataHash, strdup(mod->basePath));
mod_cache_add(mod->dataHash, 0, strdup(mod->basePath));
} else {
if (!concat_path(path, mod->basePath, mod->files[0].relativePath)) {
LOG_ERROR("Failed to combine path for mod hashing.");
return;
}
mod_cache_add(mod->dataHash, strdup(path));
mod_cache_add(mod->dataHash, 0, strdup(path));
}
}

12
src/pc/mods/mod_cache.c
View File

@ -5,7 +5,7 @@
#include "pc/debuglog.h"
#define MOD_CACHE_FILENAME "mod.cache"
#define MOD_CACHE_VERSION 1
#define MOD_CACHE_VERSION 2
struct ModCacheEntry* sModCacheHead = NULL;
@ -49,13 +49,15 @@ struct ModCacheEntry* mod_cache_get_from_path(const char* path) {
return NULL;
}
void mod_cache_add(u8* dataHash, const char* path) {
void mod_cache_add(u8* dataHash, u64 lastLoaded, const char* path) {
if (mod_cache_get_from_hash(dataHash)) {
return;
}
struct ModCacheEntry* node = calloc(1, sizeof(struct ModCacheEntry));
memcpy(node->dataHash, dataHash, 16);
if (lastLoaded == 0) { lastLoaded = clock(); }
node->lastLoaded = lastLoaded;
node->path = (char*)path;
node->next = NULL;
@ -107,17 +109,20 @@ void mod_cache_load(void) {
while (true) {
u8 dataHash[16] = { 0 };
u64 lastLoaded = 0;
u16 pathLen;
if (fread(dataHash, sizeof(u8), 16, fp) == 0) {
break;
}
fread(&lastLoaded, sizeof(u64), 1, fp);
fread(&pathLen, sizeof(u16), 1, fp);
const char* path = calloc(pathLen + 1, sizeof(u8));
fread((char*)path, sizeof(u8), pathLen + 1, fp);
mod_cache_add(dataHash, path);
mod_cache_add(dataHash, lastLoaded, path);
}
LOG_INFO("Loading mod cache complete");
@ -138,6 +143,7 @@ void mod_cache_save(void) {
struct ModCacheEntry* node = sModCacheHead;
while (node != NULL) {
fwrite(node->dataHash, sizeof(u8), 16, fp);
fwrite(node->lastLoaded, sizeof(u64), 1, fp);
u16 pathLen = strlen(node->path);
fwrite(&pathLen, sizeof(u16), 1, fp);
fwrite(node->path, sizeof(u8), pathLen + 1, fp);

3
src/pc/mods/mod_cache.h
View File

@ -4,6 +4,7 @@
struct ModCacheEntry {
u8 dataHash[16];
u64 lastLoaded;
char* path;
struct ModCacheEntry* next;
};
@ -11,7 +12,7 @@ struct ModCacheEntry {
void mod_cache_shutdown(void);
struct ModCacheEntry* mod_cache_get_from_hash(u8* dataHash);
struct ModCacheEntry* mod_cache_get_from_path(const char* path);
void mod_cache_add(u8* dataHash, const char* path);
void mod_cache_add(u8* dataHash, u64 lastLoaded, const char* path);
void mod_cache_load(void);
void mod_cache_save(void);

2
src/pc/mods/mods.c
View File

@ -62,6 +62,7 @@ void mods_activate(struct Mods* mods) {
}
// copy enabled entries
gActiveMods.entryCount = 0;
gActiveMods.size = 0;
for (int i = 0; i < mods->entryCount; i++) {
struct Mod* mod = mods->entries[i];
@ -213,5 +214,4 @@ void mods_shutdown(void) {
mods_clear(&gRemoteMods);
mods_clear(&gActiveMods);
mods_clear(&gLocalMods);
mods_delete_tmp();
}

14
src/pc/mods/mods_utils.c
View File

@ -94,20 +94,6 @@ static void mods_delete_folder(char* path) {
rmdir(path);
}
void mods_delete_tmp(void) {
// ensure tmpPath exists
char tmpPath[SYS_MAX_PATH] = { 0 };
if (snprintf(tmpPath, SYS_MAX_PATH - 1, "%s", fs_get_write_path(TMP_DIRECTORY)) < 0) {
LOG_ERROR("Failed to concat tmp path");
return;
}
// sanity
if (strlen(tmpPath) < 1) { return; }
// delete
mods_delete_folder(tmpPath);
}
//////////////////////////////////////////////////////////////////////////////////////////
bool mod_file_full_path(char* destination, struct Mod* mod, struct ModFile* modFile) {

1
src/pc/mods/mods_utils.h
View File

@ -7,7 +7,6 @@
void mods_size_enforce(struct Mods* mods);
void mods_update_selectable(void);
void mods_delete_tmp(void);
bool mod_file_full_path(char* destination, struct Mod* mod, struct ModFile* modFile);
bool mod_file_create_directories(struct Mod* mod, struct ModFile* modFile);

3
src/pc/network/network.c
View File

@ -176,6 +176,9 @@ bool network_allow_unknown_local_index(enum PacketType packetType) {
|| (packetType == PACKET_ACK)
|| (packetType == PACKET_MOD_LIST_REQUEST)
|| (packetType == PACKET_MOD_LIST)
|| (packetType == PACKET_MOD_LIST_ENTRY)
|| (packetType == PACKET_MOD_LIST_FILE)
|| (packetType == PACKET_MOD_LIST_DONE)
|| (packetType == PACKET_DOWNLOAD_REQUEST)
|| (packetType == PACKET_DOWNLOAD)
|| (packetType == PACKET_KEEP_ALIVE)

3
src/pc/network/packets/packet.c
View File

@ -83,6 +83,9 @@ void packet_process(struct Packet* p) {
case PACKET_MOD_LIST: network_receive_mod_list(p); break;
case PACKET_DOWNLOAD_REQUEST: network_receive_download_request(p); break;
case PACKET_DOWNLOAD: network_receive_download(p); break;
case PACKET_MOD_LIST_ENTRY: network_receive_mod_list_entry(p); break;
case PACKET_MOD_LIST_FILE: network_receive_mod_list_file(p); break;
case PACKET_MOD_LIST_DONE: network_receive_mod_list_done(p); break;
case PACKET_LUA_SYNC_TABLE_REQUEST: network_receive_lua_sync_table_request(p); break;
case PACKET_LUA_SYNC_TABLE: network_receive_lua_sync_table(p); break;

6
src/pc/network/packets/packet.h
View File

@ -58,6 +58,9 @@ enum PacketType {
PACKET_MOD_LIST,
PACKET_DOWNLOAD_REQUEST,
PACKET_DOWNLOAD,
PACKET_MOD_LIST_ENTRY,
PACKET_MOD_LIST_FILE,
PACKET_MOD_LIST_DONE,
PACKET_LUA_SYNC_TABLE_REQUEST,
PACKET_LUA_SYNC_TABLE,
@ -327,6 +330,9 @@ void network_send_mod_list_request(void);
void network_receive_mod_list_request(UNUSED struct Packet* p);
void network_send_mod_list(void);
void network_receive_mod_list(struct Packet* p);
void network_receive_mod_list_entry(struct Packet* p);
void network_receive_mod_list_file(struct Packet* p);
void network_receive_mod_list_done(struct Packet* p);
// packet_download.c
void network_start_download_requests(void);

2
src/pc/network/packets/packet_download.c
View File

@ -61,7 +61,7 @@ static void mark_groups_loaded_from_hash(void) {
if (mod->loadedFromCache) {
// if we loaded from cache, mark bytes as downloaded
sTotalDownloadBytes += mod->size;
LOG_INFO("Loaded from cache: %s, %lu", mod->name, mod->size);
LOG_INFO("Loaded from cache: %s, %llu", mod->name, mod->size);
} else {
// if we haven't loaded from cache, we need this offset group
u64 ogIndexStart = fileStartOffset / GROUP_SIZE;

223
src/pc/network/packets/packet_mod_list.c
View File

@ -32,6 +32,8 @@ void network_receive_mod_list_request(UNUSED struct Packet* p) {
void network_send_mod_list(void) {
SOFT_ASSERT(gNetworkType == NT_SERVER);
packet_ordered_begin();
struct Packet p = { 0 };
packet_init(&p, PACKET_MOD_LIST, true, PLMT_NONE);
@ -40,6 +42,7 @@ void network_send_mod_list(void) {
LOG_INFO("sending version: %s", version);
packet_write(&p, &version, sizeof(u8) * MAX_VERSION_LENGTH);
packet_write(&p, &gActiveMods.entryCount, sizeof(u16));
network_send_to(0, &p);
LOG_INFO("sent mod list (%u):", gActiveMods.entryCount);
for (u16 i = 0; i < gActiveMods.entryCount; i++) {
@ -51,6 +54,9 @@ void network_send_mod_list(void) {
u16 relativePathLength = strlen(mod->relativePath);
u64 modSize = mod->size;
struct Packet p = { 0 };
packet_init(&p, PACKET_MOD_LIST_ENTRY, true, PLMT_NONE);
packet_write(&p, &i, sizeof(u16));
packet_write(&p, &nameLength, sizeof(u16));
packet_write(&p, mod->name, sizeof(u8) * nameLength);
packet_write(&p, &relativePathLength, sizeof(u16));
@ -58,20 +64,32 @@ void network_send_mod_list(void) {
packet_write(&p, &modSize, sizeof(u64));
packet_write(&p, &mod->isDirectory, sizeof(u8));
packet_write(&p, &mod->dataHash[0], sizeof(u8) * 16);
packet_write(&p, &mod->fileCount, sizeof(u16));
network_send_to(0, &p);
LOG_INFO(" '%s': %llu", mod->name, (u64)mod->size);
packet_write(&p, &mod->fileCount, sizeof(u16));
for (u16 j = 0; j < mod->fileCount; j++) {
struct Packet p = { 0 };
packet_init(&p, PACKET_MOD_LIST_FILE, true, PLMT_NONE);
struct ModFile* file = &mod->files[j];
u16 relativePathLength = strlen(file->relativePath);
u64 fileSize = file->size;
packet_write(&p, &i, sizeof(u16));
packet_write(&p, &j, sizeof(u16));
packet_write(&p, &relativePathLength, sizeof(u16));
packet_write(&p, file->relativePath, sizeof(u8) * relativePathLength);
packet_write(&p, &fileSize, sizeof(u64));
network_send_to(0, &p);
LOG_INFO(" '%s': %llu", file->relativePath, (u64)file->size);
}
}
network_send_to(0, &p);
struct Packet p2 = { 0 };
packet_init(&p2, PACKET_MOD_LIST_DONE, true, PLMT_NONE);
network_send_to(0, &p2);
packet_ordered_end();
}
void network_receive_mod_list(struct Packet* p) {
@ -79,7 +97,7 @@ void network_receive_mod_list(struct Packet* p) {
if (p->localIndex != UNKNOWN_LOCAL_INDEX) {
if (gNetworkPlayerServer == NULL || gNetworkPlayerServer->localIndex != p->localIndex) {
LOG_ERROR("Received download from known local index '%d'", p->localIndex);
LOG_ERROR("Received mod list from known local index '%d'", p->localIndex);
return;
}
}
@ -118,72 +136,147 @@ void network_receive_mod_list(struct Packet* p) {
}
LOG_INFO("received mod list (%u):", gRemoteMods.entryCount);
}
void network_receive_mod_list_entry(struct Packet* p) {
SOFT_ASSERT(gNetworkType == NT_CLIENT);
// make sure it was sent by the server
if (p->localIndex != UNKNOWN_LOCAL_INDEX) {
if (gNetworkPlayerServer == NULL || gNetworkPlayerServer->localIndex != p->localIndex) {
LOG_ERROR("Received download from known local index '%d'", p->localIndex);
return;
}
}
// get mod index
u16 modIndex = 0;
packet_read(p, &modIndex, sizeof(u16));
if (modIndex >= gRemoteMods.entryCount) {
LOG_ERROR("Received mod outside of known range");
return;
}
// allocate mod entry
gRemoteMods.entries[modIndex] = calloc(1, sizeof(struct Mod));
struct Mod* mod = gRemoteMods.entries[modIndex];
if (mod == NULL) {
LOG_ERROR("Failed to allocate remote mod!");
return;
}
// get name length
u16 nameLength = 0;
packet_read(p, &nameLength, sizeof(u16));
if (nameLength > 31) {
LOG_ERROR("Received name with invalid length!");
return;
}
// get name
char name[32] = { 0 };
packet_read(p, name, nameLength * sizeof(u8));
mod->name = strdup(name);
// get other fields
u16 relativePathLength = 0;
packet_read(p, &relativePathLength, sizeof(u16));
packet_read(p, mod->relativePath, relativePathLength * sizeof(u8));
packet_read(p, &mod->size, sizeof(u64));
packet_read(p, &mod->isDirectory, sizeof(u8));
packet_read(p, &mod->dataHash, sizeof(u8) * 16);
normalize_path(mod->relativePath);
LOG_INFO(" '%s': %llu", mod->name, (u64)mod->size);
// figure out base path
if (mod->isDirectory) {
if (snprintf(mod->basePath, SYS_MAX_PATH - 1, "%s/%s", gRemoteModsBasePath, mod->relativePath) < 0) {
LOG_ERROR("Failed save remote base path!");
return;
}
normalize_path(mod->basePath);
} else {
if (snprintf(mod->basePath, SYS_MAX_PATH - 1, "%s", gRemoteModsBasePath) < 0) {
LOG_ERROR("Failed save remote base path!");
return;
}
}
// sanity check mod size
if (mod->size >= MAX_MOD_SIZE) {
djui_popup_create("Server had too large of a mod.\nQuitting.", 4);
network_shutdown(false);
return;
}
// get file count and allocate them
packet_read(p, &mod->fileCount, sizeof(u16));
mod->files = calloc(mod->fileCount, sizeof(struct ModFile));
if (mod->files == NULL) {
LOG_ERROR("Failed to allocate mod files!");
return;
}
}
void network_receive_mod_list_file(struct Packet* p) {
SOFT_ASSERT(gNetworkType == NT_CLIENT);
if (p->localIndex != UNKNOWN_LOCAL_INDEX) {
if (gNetworkPlayerServer == NULL || gNetworkPlayerServer->localIndex != p->localIndex) {
LOG_ERROR("Received download from known local index '%d'", p->localIndex);
return;
}
}
// get mod index
u16 modIndex = 0;
packet_read(p, &modIndex, sizeof(u16));
if (modIndex >= gRemoteMods.entryCount) {
LOG_ERROR("Received mod outside of known range");
return;
}
struct Mod* mod = gRemoteMods.entries[modIndex];
if (mod == NULL) {
LOG_ERROR("Received mod file for null mod");
return;
}
// get file index
u16 fileIndex = 0;
packet_read(p, &fileIndex, sizeof(u16));
if (fileIndex >= mod->fileCount) {
LOG_ERROR("Received mod file outside of known range");
return;
}
struct ModFile* file = &mod->files[fileIndex];
if (mod == NULL) {
LOG_ERROR("Received null mod file");
return;
}
u16 relativePathLength = 0;
packet_read(p, &relativePathLength, sizeof(u16));
packet_read(p, file->relativePath, relativePathLength * sizeof(u8));
packet_read(p, &file->size, sizeof(u64));
file->fp = NULL;
LOG_INFO(" '%s': %llu", file->relativePath, (u64)file->size);
}
void network_receive_mod_list_done(struct Packet* p) {
SOFT_ASSERT(gNetworkType == NT_CLIENT);
if (p->localIndex != UNKNOWN_LOCAL_INDEX) {
if (gNetworkPlayerServer == NULL || gNetworkPlayerServer->localIndex != p->localIndex) {
LOG_ERROR("Received download from known local index '%d'", p->localIndex);
return;
}
}
size_t totalSize = 0;
for (u16 i = 0; i < gRemoteMods.entryCount; i++) {
gRemoteMods.entries[i] = calloc(1, sizeof(struct Mod));
struct Mod* mod = gRemoteMods.entries[i];
if (mod == NULL) {
LOG_ERROR("Failed to allocate remote mod!");
return;
}
char name[32] = { 0 };
u16 nameLength = 0;
packet_read(p, &nameLength, sizeof(u16));
if (nameLength > 31) {
LOG_ERROR("Received name with invalid length!");
return;
}
packet_read(p, name, nameLength * sizeof(u8));
mod->name = strdup(name);
u16 relativePathLength = 0;
packet_read(p, &relativePathLength, sizeof(u16));
packet_read(p, mod->relativePath, relativePathLength * sizeof(u8));
packet_read(p, &mod->size, sizeof(u64));
packet_read(p, &mod->isDirectory, sizeof(u8));
packet_read(p, &mod->dataHash, sizeof(u8) * 16);
normalize_path(mod->relativePath);
totalSize += mod->size;
LOG_INFO(" '%s': %llu", mod->name, (u64)mod->size);
if (mod->isDirectory) {
if (snprintf(mod->basePath, SYS_MAX_PATH - 1, "%s/%s", gRemoteModsBasePath, mod->relativePath) < 0) {
LOG_ERROR("Failed save remote base path!");
return;
}
normalize_path(mod->basePath);
} else {
if (snprintf(mod->basePath, SYS_MAX_PATH - 1, "%s", gRemoteModsBasePath) < 0) {
LOG_ERROR("Failed save remote base path!");
return;
}
}
if (mod->size >= MAX_MOD_SIZE) {
djui_popup_create("Server had too large of a mod.\nQuitting.", 4);
network_shutdown(false);
return;
}
packet_read(p, &mod->fileCount, sizeof(u16));
mod->files = calloc(mod->fileCount, sizeof(struct ModFile));
if (mod->files == NULL) {
LOG_ERROR("Failed to allocate mod files!");
return;
}
for (u16 j = 0; j < mod->fileCount; j++) {
struct ModFile* file = &mod->files[j];
u16 relativePathLength = 0;
packet_read(p, &relativePathLength, sizeof(u16));
packet_read(p, file->relativePath, relativePathLength * sizeof(u8));
packet_read(p, &file->size, sizeof(u64));
file->fp = NULL;
LOG_INFO(" '%s': %llu", file->relativePath, (u64)file->size);
}
mod_load_from_cache(mod);
}
gRemoteMods.size = totalSize;

3
src/pc/network/packets/packet_reliable.c
View File

@ -125,6 +125,9 @@ static float network_adjust_max_elapsed(enum PacketType packetType, float maxEla
case PACKET_DOWNLOAD:
case PACKET_MOD_LIST_REQUEST:
case PACKET_MOD_LIST:
case PACKET_MOD_LIST_ENTRY:
case PACKET_MOD_LIST_FILE:
case PACKET_MOD_LIST_DONE:
return 0.2f + maxElapsed * 2.0f;
default:
return maxElapsed;