Adjust VRAM/RAM split on Apple Silicon

[dr3murr]

// this tool allows you to change the VRAM/RAM split on Unified Memory on Apple Silicon to whatever you want, allowing for more VRAM for inference
// c++ -std=c++17 -framework CoreFoundation -o vram_patcher vram_patcher.cpp
// credits to @asentientbot for helping me with some stuff because I never owned a Mac before
// please read the code, if you don't understand what it does don't use it
// tested on macos ventura beta 3, pattern might be different on other versions
// usage: ./vram_patcher <desired percentage of VRAM>
#include <iostream>
#include <fstream>
#include <string>
#include <vector>
#include <sys/sysctl.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/mount.h>
#include <CoreFoundation/CoreFoundation.h>

// A helper function to get the sysctl value for a given name
std::string get_sysctl(const std::string& name) {
    size_t len = 0;
    // Get the size of the value
    if (sysctlbyname(name.c_str(), nullptr, &len, nullptr, 0) == -1) {
        std::cerr << "Failed to get sysctl size for " << name << std::endl;
        return "";
    }
    // Allocate a buffer for the value
    char* buf = new char[len];
    // Get the value
    if (sysctlbyname(name.c_str(), buf, &len, nullptr, 0) == -1) {
        std::cerr << "Failed to get sysctl value for " << name << std::endl;
        delete[] buf;
        return "";
    }
    // Convert the value to a string
    std::string value(buf, len);
    delete[] buf;
    return value;
}

// A helper function to convert a CFString to a std::string
std::string CFStringToStdString(CFStringRef cfstr) {
  if (cfstr == nullptr) return "";
  CFIndex length = CFStringGetLength(cfstr);
  CFIndex maxSize = CFStringGetMaximumSizeForEncoding(length, kCFStringEncodingUTF8) + 1;
  char* buffer = new char[maxSize];
  if (CFStringGetCString(cfstr, buffer, maxSize, kCFStringEncodingUTF8)) {
    std::string result(buffer);
    delete[] buffer;
    return result;
  } else {
    delete[] buffer;
    return "";
  }
}

// A function to get the name of a disk from a path
std::string get_volume_name(const std::string& path) {
  // Create a CFURL from the path
  CFURLRef url = CFURLCreateFromFileSystemRepresentation(nullptr, (const UInt8*)path.c_str(), path.length(), false);
  if (url == nullptr) return "";

  // Get the volume URL from the path URL
  CFURLRef volumeURL = CFURLCreateCopyDeletingLastPathComponent(nullptr, url);
  CFRelease(url);
  if (volumeURL == nullptr) return "";

  // Get the volume name from the volume URL
  CFStringRef volumeName = nullptr;
  if (CFURLCopyResourcePropertyForKey(volumeURL, kCFURLVolumeNameKey, &volumeName, nullptr)) {
    CFRelease(volumeURL);
    if (volumeName == nullptr) return "";
    // Convert the volume name to a std::string
    std::string result = CFStringToStdString(volumeName);
    CFRelease(volumeName);
    return result;
  } else {
    CFRelease(volumeURL);
    return "";
  }
}

void change_float_constant(std::vector<uint8_t>& code, float new_value) {
  // Check that the code vector has enough bytes for the two instructions
  if (code.size() < 8) {
    throw std::invalid_argument("code vector is too small");
  }

  // Convert the new float value to a 32-bit unsigned integer representation
  uint32_t new_bits = *reinterpret_cast<uint32_t*>(&new_value);

  // Extract the lower and upper 16 bits of the new value
  uint16_t low_bits = new_bits & 0xffff;
  uint16_t high_bits = new_bits >> 16;

  // Encode the new value as two mov instructions
  // The first instruction is movz w8, #low_bits
  // The second instruction is movk w8, #high_bits, lsl #16
  // The opcode format is:
  // | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 9 8 | 7 6 5 4 | 3 2 1 0 |
  // | sf  0  0  1 | opc  1  0  0 | 0  0  0  0 |    hw    |     imm16     | 0  0  0  0 |   Rd   |   0  0  0  0 |
  // where sf = 0 for 32-bit register, opc = 00 for movz, 01 for movn, 10 for movk, hw = 00 for lsl #0, 01 for lsl #16, 10 for lsl #32, 11 for lsl #48, imm16 = 16-bit immediate value, Rd = destination register

  // The first instruction has sf = 0, opc = 00, hw = 00, imm16 = low_bits, Rd = 8
  uint32_t first_opcode = 0x52800000 | (low_bits << 5) | 8;

  // The second instruction has sf = 0, opc = 10, hw = 01, imm16 = high_bits, Rd = 8
  uint32_t second_opcode = 0x72800000 | (high_bits << 5) | (1 << 21) | 8;

  // Convert the opcodes to little endian bytes and overwrite the code vector
  code[0] = first_opcode & 0xff;
  code[1] = (first_opcode >> 8) & 0xff;
  code[2] = (first_opcode >> 16) & 0xff;
  code[3] = (first_opcode >> 24) & 0xff;
  code[4] = second_opcode & 0xff;
  code[5] = (second_opcode >> 8) & 0xff;
  code[6] = (second_opcode >> 16) & 0xff;
  code[7] = (second_opcode >> 24) & 0xff;
}

int main(int argc, char** argv) {
    // Check if the program is run as root
    if (getuid() != 0) {
        std::cerr << "Sorry, this program must be run as root" << std::endl;
        return 1;
    }

    // Check if the program has one argument
    if (argc != 2) {
        std::cerr << "Usage: " << argv[0] << " vram_percentage" << std::endl;
        return 1;
    }

    // Check if the argument is a valid percentage
    int percentage = std::stoi(argv[1]);
    if (percentage < 10 || percentage > 95) {
        std::cerr << "Invalid percentage: " << percentage << std::endl;
        return 1;
    }

    // Check if the CPU is Apple Silicon
    std::string cpu_brand = get_sysctl("machdep.cpu.brand_string");
    if (cpu_brand.find("Apple") == std::string::npos) {
        std::cerr << "Sorry, this program only works on Apple Silicon" << std::endl;
        return 1;
    }

    // Define the paths for the original and patched kernel collections
    std::string original_kc = "/private/var/db/KernelExtensionManagement/KernelCollections/BootKernelCollection.kc";
    std::string patched_kc = "/tmp/BootKernelCollection.kc";
    std::string final_kc = "/Library/KernelCollections/vram_patch.kc";
    std::string script_kc = "/Library/KernelCollections/complete_patch.sh";

    // Copy the original kernel collection to a temporary directory
    std::ifstream src(original_kc, std::ios::binary);
    std::ofstream dst(patched_kc, std::ios::binary);
    if (!src || !dst) {
        std::cerr << "Failed to copy the kernel collection" << std::endl;
        return 1;
    }
    dst << src.rdbuf();
    src.close();
    dst.close();

    // Read the patched kernel collection into a vector of bytes
    std::ifstream in(patched_kc, std::ios::binary);
    if (!in) {
        std::cerr << "Failed to read the patched kernel collection" << std::endl;
        return 1;
    }
    std::vector<uint8_t> data((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
    in.close();

    // Define the byte sequence to search for
    std::vector<uint8_t> target = {0x08, 0x01, 0xC0, 0xD2, 0x3F, 0x00, 0x08, 0xEB, 0xA8, 0xAA, 0x8A, 0x52, 0xA8, 0x40, 0xA8, 0x72, 0x00, 0x01, 0x27, 0x1E, 0x01, 0x30, 0x27, 0x1E, 0x28, 0x8C, 0x20, 0x1E};

    // Define the byte sequence to replace with
    std::vector<uint8_t> replacement = {0x68, 0x73, 0x89, 0x52, 0xA8, 0x94, 0xA8, 0x72, 0x08, 0x01, 0x27, 0x1E, 0x1F, 0x20, 0x03, 0xD5, 0x1F, 0x20, 0x03, 0xD5, 0x1F, 0x20, 0x03, 0xD5, 0x1F, 0x20, 0x03, 0xD5};

    // Subtract the percentage from 100
    percentage = 100 - percentage;

    // Change the floating point constant in the replacement byte sequence
    change_float_constant(replacement, percentage);

    // Find the first occurrence of the target byte sequence in the data vector
    auto it = std::search(data.begin(), data.end(), target.begin(), target.end());

    // If the target byte sequence is found, replace it with the replacement byte sequence
    if (it != data.end()) {
        std::copy(replacement.begin(), replacement.end(), it);
    } else {
        std::cerr << "Failed to find the target byte sequence in the kernel collection" << std::endl;
        return 1;
    }

    // Write the modified data vector to the patched kernel collection
    std::ofstream out(patched_kc, std::ios::binary);
    if (!out) {
        std::cerr << "Failed to write the patched kernel collection" << std::endl;
        return 1;
    }
    out.write(reinterpret_cast<char*>(data.data()), data.size());
    out.close();

    // Copy the patched kernel collection to the final destination
    std::ifstream src2(patched_kc, std::ios::binary);
    std::ofstream dst2(final_kc, std::ios::binary);
    if (!src2 || !dst2) {
        std::cerr << "Failed to copy the patched kernel collection; please disable SIP and try again" << std::endl;
        return 1;
    }
    dst2 << src2.rdbuf();
    src2.close();
    dst2.close();

    // Change the ownership of the final kernel collection to root:wheel
    if (chown(final_kc.c_str(), 0, 0) == -1) {
        std::cerr << "Failed to change the ownership of the final kernel collection" << std::endl;
        return 1;
    }    

    // Get the volume name for the /System path
    std::string volume_name = get_volume_name("/System");
    if (volume_name.empty()) {
        std::cerr << "Failed to get the volume name for /System" << std::endl;
        return 1;
    }

    // Create a shell script to configure the boot with the final kernel collection
    std::ofstream script(script_kc);
    if (!script) {
        std::cerr << "Failed to create the shell script" << std::endl;
        return 1;
    }
    script << "#!/bin/bash\n";
    script << "# Disable System Integrity Protection\n";
    script << "csrutil disable\n";
    script << "# Disable Apple Mobile File Integrity\n";
    script << "nvram boot-args=\"ipc_control_port_options=0 amfi_get_out_of_my_way=1\"\n";
    script << "# Get the absolute path of this script\n";
    script << "SCRIPT=$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\" && pwd)/$(basename \"${BASH_SOURCE[0]}\")\n";
    script << "# Append the kernel collection name to the script path\n";
    script << "KC=\"${SCRIPT%/*}/vram_patch.kc\"\n"; // This line replaces the original one
    script << "# Get the volume name from the script path\n";
    script << "VOLUME=\"${SCRIPT%%/Library*}\"\n";
    script << "# Run the kmutil command with the absolute paths\n";
    script << "kmutil configure-boot -C -c \"$KC\" -v \"$VOLUME\"\n";
    script << "sync\n";
    script << "echo Finished. You may reboot your system now.\n";
    script.close();

    // Change the ownership and permissions of the shell script to root:wheel and 755
    if (chown(script_kc.c_str(), 0, 0) == -1) {
        std::cerr << "Failed to change the ownership of the shell script" << std::endl;
        return 1;
    }
    if (chmod(script_kc.c_str(), 0755) == -1) {
        std::cerr << "Failed to change the permissions of the shell script" << std::endl;
        return 1;
    }

    // Output the instructions to run the shell script in RecoveryOS
    std::cout << "The shell script to configure the boot with the patched kernel collection has been created at " << script_kc << std::endl;
    std::cout << "Now reboot to RecoveryOS and run: \"/Volumes/" << volume_name << "/Library/KernelCollections/complete_patch.sh\"" << std::endl;
    std::cout << "If your boot fails after running the script or after an update, you need to go to Utilities->Startup Security Tool in RecoveryOS and pick Full Security" << std::endl;

    return 0;
}

[dr3murr]

if you see this screen (after an update or otherwise), click "Recovery", sign in, go to Utilities->Startup Security Tool, and click "Reduced Security"
then open terminal and run "csrutil disable" and reboot
then you can patch again

[JasonOSX]

Is this supposed to fix the shortcoming that only 2/3 or 3/4 of the unified memory can be used for Metal inference? If so, I am very eager to test it out and would like to thank you for your contribution! I am wondering if "reduced security" is only temporary. Can you enable full security after the patch? Did you also ping @ggerganov - I guess he will also be interested in this concept

[ggerganov]

Thanks - interesting approach. I'll wait on some more reports as I don't have a spare machine to test on. Would be great if this solves the memory limit issue

[dr3murr]

it def does on my machine

[dr3murr]

Yes, this fixes that. You need relaxed security or else it will boot the unpatched kernel. This is why changing the security level undoes the patch.

[JasonOSX]

Yes, this fixes that. You need relaxed security or else it will boot the unpatched kernel. This is why changing the security level undoes the patch.

Thanks for your work!

I understand that it will not boot a custom kernel with full protection, comparable to a locked (secure) bootloader on a phone.

However I would advice against people disabling or loosening their security unless they understand all the risks associated. It can open a path to sophisticated and persistent threats, especially when working in AI research as some of us are.

More info: https://support.apple.com/guide/security/startup-disk-security-policy-control-sec7d92dc49f/web

I know someone at the Apple software engineering team. I will try to ask him if they can consider upping the limit a bit or if there is a way to use a similar approach without sacrificing on security. (I guess it will be difficult but I can at least give it a shot)

[dr3murr]

vm_size_t AGXAccelerator::calcMaxGPUPhysicalMemoryBytes(uint64_t unifiedMemoryBytes) {
  float reservePercent = 33.333f;
  // Use a lower percentage if the unified memory is larger than 32 GB
  if (unifiedMemoryBytes > 0x800000000LL) {
    reservePercent = 25.0f;
  }
  const OSMetaClassBase *prop = this->device->getProperty("gpu-sysmem-reserve-percent");
  if (prop) {
    OSData *data = OSDynamicCast(OSData, prop);
    if (data) {
      reservePercent = *(float *)data->getBytesNoCopy();
    }
  }
  return (vm_size_t)((unifiedMemoryBytes * (100.0f - reservePercent) / 100.0f + page_size - 1) & -page_size);
}

that would be great
the code already has a case to check for an override %, but I can find no way to set gpu-sysmem-reserve-percent without a patch

[JasonOSX]

vm_size_t AGXAccelerator::calcMaxGPUPhysicalMemoryBytes(uint64_t unifiedMemoryBytes) {
  float reservePercent = 33.333f;
  // Use a lower percentage if the unified memory is larger than 32 GB
  if (unifiedMemoryBytes > 0x800000000LL) {
    reservePercent = 25.0f;
  }
  const OSMetaClassBase *prop = this->device->getProperty("gpu-sysmem-reserve-percent");
  if (prop) {
    OSData *data = OSDynamicCast(OSData, prop);
    if (data) {
      reservePercent = *(float *)data->getBytesNoCopy();
    }
  }
  return (vm_size_t)((unifiedMemoryBytes * (100.0f - reservePercent) / 100.0f + page_size - 1) & -page_size);
}

that would be great the code already has a case to check for an override %, but I can find no way to set gpu-sysmem-reserve-percent without a patch

That's an interesting find! but with your current approach we need both disabled SIP (protects the entire system by preventing the execution of unauthorized code) AND reduced security boot policy? what happens if you re-enable SIP but keep reduced security boot policy?

[dr3murr]

not quite sure, you need sip disabled to write to /library/kernelcollections
or at least you need it to delete from it, which is necessary if you patch again
maybe you could avoid it by writing to another path that's not sip protected but idk where
the physical file on disk isnt the actual kernel that gets booted from fwiw because you can rm or corrupt the entire file you set-boot-object'd and it will still boot

[WiseFarAI]

This is great progress! I agree that permanently disabling SIP and Full security is a risk some might not want to take too lightly. I think it would be good if some more eyes could look at this. I will also ponder about another way of implementing this change

// this tool allows you to change the VRAM/RAM split on Unified Memory on Apple Silicon to whatever you want, allowing for more VRAM for inference
// c++ -std=c++17 -framework CoreFoundation -o vram_patcher vram_patcher.cpp
// credits to @asentientbot for helping me with some stuff because I never owned a Mac before
// please read the code, if you don't understand what it does don't use it
// tested on macos ventura beta 3, pattern might be different on other versions
// usage: ./vram_patcher <desired percentage of VRAM>
#include <iostream>
#include <fstream>
#include <string>
#include <vector>
#include <sys/sysctl.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/mount.h>
#include <CoreFoundation/CoreFoundation.h>

// A helper function to get the sysctl value for a given name
std::string get_sysctl(const std::string& name) {
    size_t len = 0;
    // Get the size of the value
    if (sysctlbyname(name.c_str(), nullptr, &len, nullptr, 0) == -1) {
        std::cerr << "Failed to get sysctl size for " << name << std::endl;
        return "";
    }
    // Allocate a buffer for the value
    char* buf = new char[len];
    // Get the value
    if (sysctlbyname(name.c_str(), buf, &len, nullptr, 0) == -1) {
        std::cerr << "Failed to get sysctl value for " << name << std::endl;
        delete[] buf;
        return "";
    }
    // Convert the value to a string
    std::string value(buf, len);
    delete[] buf;
    return value;
}

// A helper function to convert a CFString to a std::string
std::string CFStringToStdString(CFStringRef cfstr) {
  if (cfstr == nullptr) return "";
  CFIndex length = CFStringGetLength(cfstr);
  CFIndex maxSize = CFStringGetMaximumSizeForEncoding(length, kCFStringEncodingUTF8) + 1;
  char* buffer = new char[maxSize];
  if (CFStringGetCString(cfstr, buffer, maxSize, kCFStringEncodingUTF8)) {
    std::string result(buffer);
    delete[] buffer;
    return result;
  } else {
    delete[] buffer;
    return "";
  }
}

// A function to get the name of a disk from a path
std::string get_volume_name(const std::string& path) {
  // Create a CFURL from the path
  CFURLRef url = CFURLCreateFromFileSystemRepresentation(nullptr, (const UInt8*)path.c_str(), path.length(), false);
  if (url == nullptr) return "";

  // Get the volume URL from the path URL
  CFURLRef volumeURL = CFURLCreateCopyDeletingLastPathComponent(nullptr, url);
  CFRelease(url);
  if (volumeURL == nullptr) return "";

  // Get the volume name from the volume URL
  CFStringRef volumeName = nullptr;
  if (CFURLCopyResourcePropertyForKey(volumeURL, kCFURLVolumeNameKey, &volumeName, nullptr)) {
    CFRelease(volumeURL);
    if (volumeName == nullptr) return "";
    // Convert the volume name to a std::string
    std::string result = CFStringToStdString(volumeName);
    CFRelease(volumeName);
    return result;
  } else {
    CFRelease(volumeURL);
    return "";
  }
}

void change_float_constant(std::vector<uint8_t>& code, float new_value) {
  // Check that the code vector has enough bytes for the two instructions
  if (code.size() < 8) {
    throw std::invalid_argument("code vector is too small");
  }

  // Convert the new float value to a 32-bit unsigned integer representation
  uint32_t new_bits = *reinterpret_cast<uint32_t*>(&new_value);

  // Extract the lower and upper 16 bits of the new value
  uint16_t low_bits = new_bits & 0xffff;
  uint16_t high_bits = new_bits >> 16;

  // Encode the new value as two mov instructions
  // The first instruction is movz w8, #low_bits
  // The second instruction is movk w8, #high_bits, lsl #16
  // The opcode format is:
  // | 31 30 29 28 | 27 26 25 24 | 23 22 21 20 | 19 18 17 16 | 15 14 13 12 | 11 10 9 8 | 7 6 5 4 | 3 2 1 0 |
  // | sf  0  0  1 | opc  1  0  0 | 0  0  0  0 |    hw    |     imm16     | 0  0  0  0 |   Rd   |   0  0  0  0 |
  // where sf = 0 for 32-bit register, opc = 00 for movz, 01 for movn, 10 for movk, hw = 00 for lsl #0, 01 for lsl #16, 10 for lsl #32, 11 for lsl #48, imm16 = 16-bit immediate value, Rd = destination register

  // The first instruction has sf = 0, opc = 00, hw = 00, imm16 = low_bits, Rd = 8
  uint32_t first_opcode = 0x52800000 | (low_bits << 5) | 8;

  // The second instruction has sf = 0, opc = 10, hw = 01, imm16 = high_bits, Rd = 8
  uint32_t second_opcode = 0x72800000 | (high_bits << 5) | (1 << 21) | 8;

  // Convert the opcodes to little endian bytes and overwrite the code vector
  code[0] = first_opcode & 0xff;
  code[1] = (first_opcode >> 8) & 0xff;
  code[2] = (first_opcode >> 16) & 0xff;
  code[3] = (first_opcode >> 24) & 0xff;
  code[4] = second_opcode & 0xff;
  code[5] = (second_opcode >> 8) & 0xff;
  code[6] = (second_opcode >> 16) & 0xff;
  code[7] = (second_opcode >> 24) & 0xff;
}

int main(int argc, char** argv) {
    // Check if the program is run as root
    if (getuid() != 0) {
        std::cerr << "Sorry, this program must be run as root" << std::endl;
        return 1;
    }

    // Check if the program has one argument
    if (argc != 2) {
        std::cerr << "Usage: " << argv[0] << " vram_percentage" << std::endl;
        return 1;
    }

    // Check if the argument is a valid percentage
    int percentage = std::stoi(argv[1]);
    if (percentage < 10 || percentage > 95) {
        std::cerr << "Invalid percentage: " << percentage << std::endl;
        return 1;
    }

    // Check if the CPU is Apple Silicon
    std::string cpu_brand = get_sysctl("machdep.cpu.brand_string");
    if (cpu_brand.find("Apple") == std::string::npos) {
        std::cerr << "Sorry, this program only works on Apple Silicon" << std::endl;
        return 1;
    }

    // Define the paths for the original and patched kernel collections
    std::string original_kc = "/private/var/db/KernelExtensionManagement/KernelCollections/BootKernelCollection.kc";
    std::string patched_kc = "/tmp/BootKernelCollection.kc";
    std::string final_kc = "/Library/KernelCollections/vram_patch.kc";
    std::string script_kc = "/Library/KernelCollections/complete_patch.sh";

    // Copy the original kernel collection to a temporary directory
    std::ifstream src(original_kc, std::ios::binary);
    std::ofstream dst(patched_kc, std::ios::binary);
    if (!src || !dst) {
        std::cerr << "Failed to copy the kernel collection" << std::endl;
        return 1;
    }
    dst << src.rdbuf();
    src.close();
    dst.close();

    // Read the patched kernel collection into a vector of bytes
    std::ifstream in(patched_kc, std::ios::binary);
    if (!in) {
        std::cerr << "Failed to read the patched kernel collection" << std::endl;
        return 1;
    }
    std::vector<uint8_t> data((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
    in.close();

    // Define the byte sequence to search for
    std::vector<uint8_t> target = {0x08, 0x01, 0xC0, 0xD2, 0x3F, 0x00, 0x08, 0xEB, 0xA8, 0xAA, 0x8A, 0x52, 0xA8, 0x40, 0xA8, 0x72, 0x00, 0x01, 0x27, 0x1E, 0x01, 0x30, 0x27, 0x1E, 0x28, 0x8C, 0x20, 0x1E};

    // Define the byte sequence to replace with
    std::vector<uint8_t> replacement = {0x68, 0x73, 0x89, 0x52, 0xA8, 0x94, 0xA8, 0x72, 0x08, 0x01, 0x27, 0x1E, 0x1F, 0x20, 0x03, 0xD5, 0x1F, 0x20, 0x03, 0xD5, 0x1F, 0x20, 0x03, 0xD5, 0x1F, 0x20, 0x03, 0xD5};

    // Subtract the percentage from 100
    percentage = 100 - percentage;

    // Change the floating point constant in the replacement byte sequence
    change_float_constant(replacement, percentage);

    // Find the first occurrence of the target byte sequence in the data vector
    auto it = std::search(data.begin(), data.end(), target.begin(), target.end());

    // If the target byte sequence is found, replace it with the replacement byte sequence
    if (it != data.end()) {
        std::copy(replacement.begin(), replacement.end(), it);
    } else {
        std::cerr << "Failed to find the target byte sequence in the kernel collection" << std::endl;
        return 1;
    }

    // Write the modified data vector to the patched kernel collection
    std::ofstream out(patched_kc, std::ios::binary);
    if (!out) {
        std::cerr << "Failed to write the patched kernel collection" << std::endl;
        return 1;
    }
    out.write(reinterpret_cast<char*>(data.data()), data.size());
    out.close();

    // Copy the patched kernel collection to the final destination
    std::ifstream src2(patched_kc, std::ios::binary);
    std::ofstream dst2(final_kc, std::ios::binary);
    if (!src2 || !dst2) {
        std::cerr << "Failed to copy the patched kernel collection" << std::endl;
        return 1;
    }
    dst2 << src2.rdbuf();
    src2.close();
    dst2.close();

    // Change the ownership of the final kernel collection to root:wheel
    if (chown(final_kc.c_str(), 0, 0) == -1) {
        std::cerr << "Failed to change the ownership of the final kernel collection" << std::endl;
        return 1;
    }    

    // Get the volume name for the /System path
    std::string volume_name = get_volume_name("/System");
    if (volume_name.empty()) {
        std::cerr << "Failed to get the volume name for /System" << std::endl;
        return 1;
    }

    // Create a shell script to configure the boot with the final kernel collection
    std::ofstream script(script_kc);
    if (!script) {
        std::cerr << "Failed to create the shell script" << std::endl;
        return 1;
    }
    script << "#!/bin/bash\n";
    script << "# Disable System Integrity Protection\n";
    script << "csrutil disable\n";
    script << "# Disable Apple Mobile File Integrity\n";
    script << "nvram boot-args=\"ipc_control_port_options=0 amfi_get_out_of_my_way=1\"\n";
    script << "# Get the absolute path of this script\n";
    script << "SCRIPT=$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\" && pwd)/$(basename \"${BASH_SOURCE[0]}\")\n";
    script << "# Append the kernel collection name to the script path\n";
    script << "KC=\"${SCRIPT%/*}/vram_patch.kc\"\n"; // This line replaces the original one
    script << "# Get the volume name from the script path\n";
    script << "VOLUME=\"${SCRIPT%%/Library*}\"\n";
    script << "# Run the kmutil command with the absolute paths\n";
    script << "kmutil configure-boot -C -c \"$KC\" -v \"$VOLUME\"\n";
    script << "sync\n";
    script << "echo Finished. You may reboot your system now.\n";
    script.close();

    // Change the ownership and permissions of the shell script to root:wheel and 755
    if (chown(script_kc.c_str(), 0, 0) == -1) {
        std::cerr << "Failed to change the ownership of the shell script" << std::endl;
        return 1;
    }
    if (chmod(script_kc.c_str(), 0755) == -1) {
        std::cerr << "Failed to change the permissions of the shell script" << std::endl;
        return 1;
    }

    // Output the instructions to run the shell script in RecoveryOS
    std::cout << "The shell script to configure the boot with the patched kernel collection has been created at " << script_kc << std::endl;
    std::cout << "Now reboot to RecoveryOS and run: \"/Volumes/" << volume_name << "/Library/KernelCollections/complete_patch.sh\"" << std::endl;
    std::cout << "If your boot fails after running the script or after an update, you need to go to Utilities->Startup Security Tool in RecoveryOS and pick Full Security" << std::endl;

    return 0;
}

[CyborgArmy83]

not quite sure, you need sip disabled to write to /library/kernelcollections or at least you need it to delete from it, which is necessary if you patch again maybe you could avoid it by writing to another path that's not sip protected but idk where the physical file on disk isnt the actual kernel that gets booted from fwiw because you can rm or corrupt the entire file you set-boot-object'd and it will still boot

Hey @r3muxd

As you might know the new MacOS 14 (Sonoma) Public Beta 3 has been released and includes some updated CoreML and other libraries which are used in AI development. I had to update to the 14.0 beta to work on some AI projects that required the updated frameworks.

Do you think this script works on MacOS 14 Beta or will this code need an update? I will be happy to help/test and figure it out with you! However, I can use the help and you already have some expertise from creating this in the first place :)

I tried the script before on my older Macbook Pro M1 Pro and it seemed to work but I am just a bit on the cautious side with my new M1 Max MBP.

[dr3murr]

just try it, worst come to worst you'll need to flip back on security when you get booted to recoveryos if theres an error if the pattern is wrong for new MacOS, then you'll get an error running the script and nothing will happen

…

On Monday, August 14, 2023, CyborgArmy83 ***@***.***> wrote: not quite sure, you need sip disabled to write to /library/kernelcollections or at least you need it to delete from it, which is necessary if you patch again maybe you could avoid it by writing to another path that's not sip protected but idk where the physical file on disk isnt the actual kernel that gets booted from fwiw because you can rm or corrupt the entire file you set-boot-object'd and it will still boot Hey @r3muxd <https://github.com/r3muxd> As you might know the new MacOS 14 (Sonoma) Public Beta 3 has been released and includes some updated CoreML and other libraries which are used in AI development. I had to update to the 14.0 beta to work on some AI projects that required the updated frameworks. Do you think this script works on MacOS 14 Beta or will this code need an update? I will be happy to help/test and figure it out with you! However, I can use the help and you already have some expertise from creating this in the first place :) I tried the script before on my older Macbook Pro M1 Pro and it seemed to work but I am just a bit on the cautious side with my new M1 Max MBP. — Reply to this email directly, view it on GitHub <#2182 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AK7NTZTI4ZZGM7Q42GVXMJTXVHW2PANCNFSM6AAAAAA2GNVSEI> . You are receiving this because you were mentioned.Message ID: ***@***.***>

[crasm]

@r3muxd I’m having some trouble following the instructions for my system on the latest (stable) macOS Ventura. When I boot into recovery, the directory /Volumes/Macintosh HD/Library is empty.

The KernelCollections directory on my system is available at /Volumes/Data/Library/KernelCollections after mounting the Macintosh HD - Data volume. However, reading the script, it seems unlikely the system will boot if it’s not on the main volume? I am not familiar with the macOS boot process.

Did you need to do any additional steps after booting into recovery, before running the script?

Thanks for any help! I’m looking forward to running Q4_K_M falcon 180B if I can get this to work.

[dr3murr]

In any case, it should be fine to just attempt to run the script from the other location. If MacOS fails to boot, you will get a prompt telling you to remove the custom kernel, which you can do trivially by clicking Recovery->Utilities->Startup Security Tool and resetting it to default.

[crasm]

Success! I was able to load and run inference on Q4_K Falcon 180B with my 128GB Mac Studio.

For anybody else trying to get this to work:

• diskutil list, look for a line like 5: APFS Volume Data 1.4 TB disk3s5
• Mount that volume with diskutil apfs unlock /dev/<volume>
• Run /Volumes/Data/Library/KernelCollections/complete_patch.sh

---

@r3muxd Thank you very much for your help and assurances

Notes:

• The output of your program is Macintosh HD normally and macOS Base System in recovery.
• There are some warnings and errors compiling vram_patcher.cpp, but it works fine
• ./complete_patch.sh in recovery gave an error, but it works fine
  • Error setting variable – 'boot-args': (iokit/common) not permitted.

[hlo-world]

Thanks crasm, can confirm it works on Sonoma

[JasonOSX]

Did you patch it through recovery or else? and did you disable SIP or more? asking because I just upgraded to the latest Sonoma

[hlo-world]

Yea you have to reduce the security and disable SIP in Recovery.

[ddh0]

Hi, thanks for this. I'm getting this error: Failed to copy the patched kernel collection. Reading the code I see it's just copying a file so I'm not sure what's up. I've checked to make sure both the parent directories exist, and I'm running the program as root.

I'm on macOS Sonoma 14.1, M2 Air 24GB. Trying to set percent to 87.5. Any help would be appreciated. Thanks.

[crasm]

Did you disable SIP?

[ddh0]

Ah, ok. That wasn't mentioned. Thank you!

[ddh0]

Works great now! Running mistral 7b FP16 on metal which I couldn't quite do before.

[kdyke]

Folks... just do: sudo sysctl iogpu.wired_limit_mb=<mb> from Terminal. You’d have to do it every boot as it’s not sticky, but that seems better than patching your OS and disabling SIP. I would not recommend going to 100% as the OS needs some reasonable amount of memory to fit everything else that has to be around not wired down by the GPU driver stack, and things will start to go seriously wrong if you don’t leave enough space.

[sammcj]

Ohhhh I see, thank you! :)

[ingochris]

Thank you so much!

[crasm]

For nix-darwin, you can use:

    launchd.daemons."sysctl-vram-limit" = {
      command = "/usr/sbin/sysctl iogpu.wired_limit_mb=115200";
      serviceConfig.RunAtLoad = true;
    };

[tusing]

Thanks for the Nix instructions, did not realize this is why it wasn't working!

[beginor]

indeed, it works!

[kdyke]

It’s expecting a value in megabytes. So if you’re using RAM math instead of HD/SDD math, for 60GB you’d want 60*1024 = 61440

…

-Ken

On Nov 29, 2023, at 11:48 AM, CyborgArmy83 ***@***.***> wrote: For those who've already tried this What does this line sudo sysctl iogpu.wired_limit_mb= expect? is the value within the brackets on the end a percentage value like 0-100% or a number in mb? like "60000" for 60GB — Reply to this email directly, view it on GitHub <#2182 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AH24SEDLJZZ3CNWUIWEFSCLYG6GRBAVCNFSM6AAAAAA2GNVSEKVHI2DSMVQWIX3LMV43SRDJONRXK43TNFXW4Q3PNVWWK3TUHM3TOMBZGI4DA>. You are receiving this because you were mentioned.

[ddh0]

mine was base 10 math, where 20000 was 20GB

[marcingomulkiewicz]

(Sorry for necrothreading, but I've spent some serious time looking for the solution for this particular problem, and I think I found the easiest one)

What worked for me (Sonoma) is to create sysctl.conf, and add appropriate parameter to it:

sudo nano /etc/sysctl.conf

and inside:

# change default CPU/GPU RAM split
iogpu.wired_limit_mb=28672

(as already mentioned, the number is max RAM in MB that GPU can have; select some value reasonable for your machine).

I'm not sure why /etc/sysctl.conf does not exist by default (good old unix habit is to put some template with commented out defaults), but as far as I know sysctl is not deprecated, so it should keep working.

[Summit1122]

how can I check if this worked?

[beginor]

Yes, it works! Just run any model with llama-cli or llama-server , check the ggml_metal_init: recommendedMaxWorkingSetSize value in console output, which is effected by iogpu.wired_limit_mb.

[trevcodner]

Morning, can I confirm, that this works instead of the script above, or requires that script too? I suspect it's just the command line, but wanted to check. Thanks

[jared-krauss]

(Sorry for necrothreading, but I've spent some serious time looking for the solution for this particular problem, and I think I found the easiest one)

What worked for me (Sonoma) is to create sysctl.conf, and add appropriate parameter to it:

sudo nano /etc/sysctl.conf

and inside:

# change default CPU/GPU RAM split
iogpu.wired_limit_mb=28672

(as already mentioned, the number is max RAM in MB that GPU can have; select some value reasonable for your machine).

I'm not sure why /etc/sysctl.conf does not exist by default (good old unix habit is to put some template with commented out defaults), but as far as I know sysctl is not deprecated, so it should keep working.

No no, it's appreciated. Because I too am bringing this back a year later to say this is what I needed to help me make bigger Gaussian Splats, running out of space and getting the Sigkill code.

[Azirine]

I found that even after doing sudo nano /etc/sysctl.conf, going slightly above the original limit (¾ × 128 = 96 GiB in my case) makes models load much slower with a long swap/unswap process. #13361