Avoid loading 15 MB files into RAM ever

include/fs.h

@@ -66,6 +66,7 @@ u32 decompress_lz_file(FS_Path file_name, FS_Archive archive, char ** buf);
 u32 compress_lz_file_fast(FS_Path path, FS_Archive archive, char * in_buf, u32 size);
 
 Result buf_to_file(u32 size, FS_Path path, FS_Archive archive, char * buf);
+Result zero_handle_memeasy(Handle handle);
 void remake_file(FS_Path path, FS_Archive archive, u32 size);
 void save_zip_to_sd(char * filename, u32 size, char * buf, RemoteMode mode);
 s16 for_each_file_zip(u16 *zip_path, u32 (*zip_iter_callback)(char *filebuf, u64 file_size, const char *name, void *userdata), void *userdata);

source/badges.c

@@ -33,7 +33,7 @@
 #include "ui_strings.h"
 
 static Handle actHandle;
-char *badgeDataBuffer;
+Handle badgeDataHandle;
 char *badgeMngBuffer;
 u16 *rgb_buf_64x64;
 u16 *rgb_buf_32x32;
@@ -131,12 +131,12 @@ int install_badge_generic(char *file_buf, u64 file_size, u16 *name, int *badge_c
         remove_exten(name);
         for (int j = 0; j < 16; ++j) // Copy name for all 16 languages
         {
-            memcpy(badgeDataBuffer + 0x35E80 + *badge_count * 16 * 0x8A + j * 0x8A, name, 0x8A);
+            FSFILE_Write(badgeDataHandle, NULL, 0x35E80 + *badge_count * 16 * 0x8A + j * 0x8A, name, 0x8A, 0);
         }
-        memcpy(badgeDataBuffer + 0x318F80 + *badge_count * 0x2800, rgb_buf_64x64 + badge * 64 * 64, 64 * 64 * 2);
+        FSFILE_Write(badgeDataHandle, NULL, 0x318F80 + *badge_count * 0x2800, rgb_buf_64x64 + badge * 64 * 64, 64 * 64 * 2, 0);
-        memcpy(badgeDataBuffer + 0x31AF80 + *badge_count * 0x2800, alpha_buf_64x64 + badge * 64 * 64/2, 64 * 64/2);
+        FSFILE_Write(badgeDataHandle, NULL, 0x31AF80 + *badge_count * 0x2800, alpha_buf_64x64 + badge * 64 * 64/2, 64 * 64/2, 0);
-        memcpy(badgeDataBuffer + 0xCDCF80 + *badge_count * 0xA00, rgb_buf_32x32 + badge * 32 * 32, 32 * 32 * 2);
+        FSFILE_Write(badgeDataHandle, NULL, 0xCDCF80 + *badge_count * 0xA00, rgb_buf_32x32 + badge * 32 * 32, 32 * 32 * 2, 0);
-        memcpy(badgeDataBuffer + 0xCDD780 + *badge_count * 0xA00, alpha_buf_32x32 + badge * 32 * 32/2, 32 * 32/2);
+        FSFILE_Write(badgeDataHandle, NULL, 0xCDD780 + *badge_count * 0xA00, alpha_buf_32x32 + badge * 32 * 32/2, 32 * 32/2, 0);
 
         int badge_id = *badge_count + 1;
         memcpy(badgeMngBuffer + 0x3E8 + *badge_count * 0x28 + 0x4, &badge_id, 4);
@@ -167,7 +167,9 @@ int install_badge_png(FS_Path badge_path, FS_DirectoryEntry badge_file, int *bad
         return -1;
     }
 
-    return install_badge_generic(file_buf, badge_file.fileSize, badge_file.name, badge_count, set_id);
+    int badges = install_badge_generic(file_buf, badge_file.fileSize, badge_file.name, badge_count, set_id);
+    free(file_buf);
+    return badges;
 }
 
 typedef struct {
@@ -255,7 +257,7 @@ int install_badge_dir(FS_DirectoryEntry set_dir, int *badge_count, int set_id)
     u32 total_count = 0xFFFF * badges_in_set;
     for (int i = 0; i < 16; ++i)
     {
-        strucat((u16 *) (badgeDataBuffer + set_index * 0x8A0 + i * 0x8A), set_dir.name);
+        FSFILE_Write(badgeDataHandle, NULL, set_index * 0x8A0 + i * 0x8A, set_dir.name, 0x8A, 0);
     }
     badgeMngBuffer[0x3D8 + set_index/8] |= 0 << (set_index % 8);
 
@@ -286,7 +288,7 @@ int install_badge_dir(FS_DirectoryEntry set_dir, int *badge_count, int set_id)
     }
 
     free(icon_buf);
-    memcpy(badgeDataBuffer + 0x250F80 + set_index * 0x2000, rgb_buf_64x64, 64 * 64 * 2);
+    FSFILE_Write(badgeDataHandle, NULL, 0x250F80 + set_index * 0x2000, rgb_buf_64x64, 64 * 64 * 2, 0);
     badgeMngBuffer[0x3D8 + set_index/8] |= 0 << (set_index % 8);
     end:
     free(badge_files);
@@ -296,44 +298,60 @@ int install_badge_dir(FS_DirectoryEntry set_dir, int *badge_count, int set_id)
 Result backup_badges(void)
 {
     char *badgeMng = NULL;
-    char *badgeData = NULL;
 
     DEBUG("writing badge data: making files...\n");
     remake_file(fsMakePath(PATH_ASCII, "/3ds/Anemone3DS/BadgeMngFile.dat"), ArchiveSD, BADGE_MNG_SIZE);
-    remake_file(fsMakePath(PATH_ASCII, "/3ds/Anemone3DS/BadgeData.dat"), ArchiveSD, BADGE_DATA_SIZE);
+    Handle dataHandle = 0;
+    Handle sdHandle = 0;
+    if (R_SUCCEEDED(FSUSER_OpenFile(&sdHandle, ArchiveSD, fsMakePath(PATH_ASCII, "/3ds/Anemone3DS/BadgeData.dat"), FS_OPEN_READ, 0)))
+    {
+        FSFILE_Close(sdHandle);
+        FSUSER_DeleteFile(ArchiveSD, fsMakePath(PATH_ASCII, "/3ds/Anemone3DS/BadgeData.dat"));
+    }
+    FSUSER_CreateFile(ArchiveSD, fsMakePath(PATH_ASCII, "/3ds/Anemone3DS/BadgeData.dat"), 0, BADGE_DATA_SIZE);
+    FSUSER_OpenFile(&sdHandle, ArchiveSD, fsMakePath(PATH_ASCII, "/3ds/Anemone3DS/BadgeData.dat"), FS_OPEN_WRITE, 0);
+    zero_handle_memeasy(sdHandle);
 
     DEBUG("loading existing badge mng file...\n");
     u32 mngRead = file_to_buf(fsMakePath(PATH_ASCII, "/BadgeMngFile.dat"), ArchiveBadgeExt, &badgeMng);
     DEBUG("loading existing badge data file\n");
-    u32 dataRead = file_to_buf(fsMakePath(PATH_ASCII, "/BadgeData.dat"), ArchiveBadgeExt, &badgeData);
+    Result res = FSUSER_OpenFile(&dataHandle, ArchiveBadgeExt, fsMakePath(PATH_ASCII, "/BadgeData.dat"), FS_OPEN_READ, 0);
-    if (mngRead != BADGE_MNG_SIZE || dataRead != BADGE_DATA_SIZE)
+    if (mngRead != BADGE_MNG_SIZE || R_FAILED(res))
     {
         throw_error(language.badges.extdata_locked, ERROR_LEVEL_WARNING);
         if (badgeMng) free(badgeMng);
-        if (badgeData) free(badgeData);
+        if (dataHandle) FSFILE_Close(dataHandle);
+        FSFILE_Close(sdHandle);
         return -1;
     }
 
     DEBUG("writing badge data: writing BadgeMngFile...\n");
-    Result res = buf_to_file(mngRead, fsMakePath(PATH_ASCII, "/3ds/Anemone3DS/BadgeMngFile.dat"), ArchiveSD, badgeMng);
+    res = buf_to_file(mngRead, fsMakePath(PATH_ASCII, "/3ds/Anemone3DS/BadgeMngFile.dat"), ArchiveSD, badgeMng);
     if (R_FAILED(res))
     {
         DEBUG("Failed to write badgemngfile: 0x%08lx\n", res);
         free(badgeMng);
-        free(badgeData);
+        FSFILE_Close(dataHandle);
+        FSFILE_Close(sdHandle);
         return -1;
     }
     DEBUG("writing badge data: writing badgedata...\n");
-    res = buf_to_file(dataRead, fsMakePath(PATH_ASCII, "/3ds/Anemone3DS/BadgeData.dat"), ArchiveSD, badgeData);
+    char *buf = malloc(0x10000);
-    if (R_FAILED(res))
+    u64 size = BADGE_DATA_SIZE;
+    u64 cur = 0;
+    while (size > 0)
     {
-        DEBUG("Failed to write badgedatafile: 0x%08lx\n", res);
+        u32 read = 0;
-        free(badgeMng);
+        res = FSFILE_Read(dataHandle, &read, cur, buf, min(0x10000, size));
-        free(badgeData);
+        res = FSFILE_Write(sdHandle, NULL, cur, buf, read, FS_WRITE_FLUSH);
-        return -1;
+        size -= read;
+        cur += read;
     }
+
     free(badgeMng);
-    free(badgeData);
+    free(buf);
+    FSFILE_Close(dataHandle);
+    FSFILE_Close(sdHandle);
     return 0;
 }
 
@@ -342,6 +360,7 @@ Result install_badges(void)
     Handle handle = 0;
     Handle folder = 0;
     Result res = 0;
+    draw_loading_bar(0, 1, INSTALL_BADGES);
 
     DEBUG("Backing up badges\n");
     res = backup_badges();
@@ -377,7 +396,7 @@ Result install_badges(void)
     }
 
     badgeMngBuffer = NULL;
-    badgeDataBuffer = NULL;
+    badgeDataHandle = 0;
     rgb_buf_64x64 = NULL;
     rgb_buf_32x32 = NULL;
     alpha_buf_64x64 = NULL;
@@ -399,13 +418,12 @@ Result install_badges(void)
     alpha_buf_64x64 = malloc(12*6*64*64/2); //Same thing, but 2 pixels of alpha data per byte
     rgb_buf_32x32 = malloc(12*6*32*32*2); //Same thing, but 32x32
     alpha_buf_32x32 = malloc(12*6*32*32/2);
-    badgeDataBuffer = calloc(1, BADGE_DATA_SIZE);
+    res = FSUSER_OpenFile(&badgeDataHandle, ArchiveBadgeExt, fsMakePath(PATH_ASCII, "/BadgeData.dat"), FS_OPEN_WRITE, 0);
     badgeMngBuffer = calloc(1, BADGE_MNG_SIZE);
 
     if (!rgb_buf_64x64)
     {
         DEBUG("rgb_buf_64x64 alloc failed\n");
-        res = -1;
         goto end;
     }
 
@@ -427,9 +445,11 @@ Result install_badges(void)
         goto end;
     }
 
-    if (!badgeDataBuffer)
+    if (R_FAILED(res))
     {
-        DEBUG("badgeDataBuffer alloc failed\n");
+        badgeDataHandle = 0;
+        throw_error(language.badges.extdata_locked, ERROR_LEVEL_WARNING);
+        DEBUG("badgeDataHandle open failed\n");
         goto end;
     }
 
@@ -439,17 +459,19 @@ Result install_badges(void)
         goto end;
     }
 
+    zero_handle_memeasy(badgeDataHandle);
+
     int badge_count = 0;
     int set_count = 0;
     int default_set = 0;
     int default_set_count = 0;
     int default_idx = 0;
-    draw_loading_bar(0, entries_read + 4, INSTALL_BADGES);
+
+    draw_loading_bar(6, entries_read + 12, INSTALL_BADGES);
     for (u32 i = 0; i < entries_read && badge_count < 1000; ++i)
     {
         if (!strcmp(badge_files[i].shortExt, "PNG"))
         {
-            DEBUG("PNG discovered\n");
             if (default_set == 0)
             {
                 set_count += 1;
@@ -462,7 +484,6 @@ Result install_badges(void)
             default_set_count += install_badge_png(fsMakePath(PATH_UTF16, path), badge_files[i], &badge_count, default_set);
         } else if (!strcmp(badge_files[i].shortExt, "ZIP"))
         {
-            DEBUG("ZIP discovered\n");
             if (default_set == 0)
             {
                 set_count += 1;
@@ -475,13 +496,12 @@ Result install_badges(void)
             default_set_count += install_badge_zip(path, &badge_count, default_set);
         } else if (badge_files[i].attributes & FS_ATTRIBUTE_DIRECTORY)
         {
-            DEBUG("dir discovered\n");
             set_count += 1;
             u32 count = install_badge_dir(badge_files[i], &badge_count, set_count);
             if (count == 0)
                 set_count -= 1;
         }
-        draw_loading_bar(i, entries_read + 4, INSTALL_BADGES);
+        draw_loading_bar(i + 7, entries_read + 12, INSTALL_BADGES);
     }
 
     DEBUG("Badges installed - doing metadata\n");
@@ -493,7 +513,7 @@ Result install_badges(void)
         {
             u16 name[0x8A/2] = {0};
             utf8_to_utf16(name, (u8 *) "Other Badges", 0x8A);
-            memcpy(badgeDataBuffer + default_index * 0x8A0 + i * 0x8A, &name, 0x8A);
+            FSFILE_Write(badgeDataHandle, NULL, default_index * 0x8A0 + i * 0x8A, &name, 0x8A, 0);
         }
         badgeMngBuffer[0x3D8 + default_index/8] |= 0 << (default_index % 8);
 
@@ -516,16 +536,11 @@ Result install_badges(void)
         fclose(fp);
         pngToRGB565(icon_buf, size, rgb_buf_64x64, alpha_buf_64x64, rgb_buf_32x32, alpha_buf_32x32, true);
         free(icon_buf);
-        memcpy(badgeDataBuffer + 0x250F80 + default_index * 0x2000, rgb_buf_64x64, 64 * 64 * 2);
+        FSFILE_Write(badgeDataHandle, NULL, 0x250F80 + default_index * 0x2000, rgb_buf_64x64, 64 * 64 * 2, 0);
     }
 
-    res = buf_to_file(BADGE_DATA_SIZE, fsMakePath(PATH_ASCII, "/BadgeData.dat"), ArchiveBadgeExt, badgeDataBuffer);
+    FSFILE_Flush(badgeDataHandle);
-    if (res)
+    draw_loading_bar(entries_read + 9, entries_read + 12, INSTALL_BADGES);
-    {
-        DEBUG("Error writing badge data! %lx\n", res);
-        throw_error(language.badges.extdata_locked, ERROR_LEVEL_WARNING);
-        goto end;
-    }
 
     u32 total_badges = 0xFFFF * badge_count; // Quantity * unique badges?
 
@@ -575,7 +590,7 @@ Result install_badges(void)
     if (alpha_buf_32x32) free(alpha_buf_32x32);
     if (handle) FSFILE_Close(handle);
     if (folder) FSDIR_Close(folder);
-    if (badgeDataBuffer) free(badgeDataBuffer);
+    if (badgeDataHandle) FSFILE_Close(badgeDataHandle);
     if (badgeMngBuffer) free(badgeMngBuffer);
     if (badge_files) free(badge_files);
     return res;

source/fs.c

@@ -151,7 +151,11 @@ Result open_badge_extdata()
             createExtSaveData(0x000014d1);
             FSUSER_OpenArchive(&ArchiveBadgeExt, ARCHIVE_EXTDATA, badge);
             remake_file(fsMakePath(PATH_ASCII, "/BadgeMngFile.dat"), ArchiveBadgeExt, BADGE_MNG_SIZE);
-            remake_file(fsMakePath(PATH_ASCII, "/BadgeData.dat"), ArchiveBadgeExt, BADGE_DATA_SIZE);
+            FSUSER_CreateFile(ArchiveBadgeExt, fsMakePath(PATH_ASCII, "/BadgeData.dat"), 0, BADGE_DATA_SIZE);
+            FSUSER_OpenFile(&test_handle, ArchiveBadgeExt, fsMakePath(PATH_ASCII, "/BadgeData.dat"), FS_OPEN_WRITE, 0);
+            zero_handle_memeasy(test_handle);
+            FSFILE_Flush(test_handle);
+            FSFILE_Close(test_handle);
         } else
         {
             return res;
@@ -488,6 +492,23 @@ void remake_file(FS_Path path, FS_Archive archive, u32 size)
     }
 }
 
+Result zero_handle_memeasy(Handle handle)
+{
+    u64 size = 0;
+    u64 cur = 0;
+    FSFILE_GetSize(handle, &size);
+    char *zero_buf = calloc(1, 0x10000);
+    while (size > 0x10000)
+    {
+        FSFILE_Write(handle, NULL, cur, &zero_buf, 0x1000, 0);
+        cur += 0x10000;
+        size -= 0x10000;
+    }
+    FSFILE_Write(handle, NULL, cur, &zero_buf, size, 0);
+    free(zero_buf);
+    return 0;
+}
+
 static SwkbdCallbackResult fat32filter(void * user, const char ** ppMessage, const char * text, size_t textlen)
 {
     (void)textlen;