VirtualMemory.c

Go to the documentation of this file.

 
#include <Uefi.h>
 
#include <Library/UefiLib.h>
#include <Library/BaseMemoryLib.h>
#include <Library/BaseOverflowLib.h>
#include <Library/MtrrLib.h>
#include <Library/OcDebugLogLib.h>
#include <Library/OcMemoryLib.h>
 
#include <Register/Intel/Cpuid.h>
 
//
// Index on MTRR value.
// (ref. table 11-8)
//
STATIC CHAR8  *MtrrTypeStrings[] = {
  "UC",
  "WC",
  "Reserved",
  "Reserved",
  "WT",
  "WP",
  "WB"
};
 
//
// Index on PAT type in PAT MSR register, which itself is
// indexed on PAT bits from page table.
// (ref. table 11-10)
//
STATIC CHAR8  *PatTypeStrings[] = {
  "UC",
  "WC",
  "Reserved",
  "Reserved",
  "WT",
  "WP",
  "WB",
  "UC-"
};
 
CHAR8 *
OcGetMtrrTypeString (
  UINT8  MtrrType
  )
{
  if (MtrrType >= ARRAY_SIZE (MtrrTypeStrings)) {
    return "Invalid";
  } else {
    return MtrrTypeStrings[MtrrType];
  }
}
 
CHAR8 *
OcGetPatTypeString (
  UINT8  PatType
  )
{
  if (PatType >= ARRAY_SIZE (PatTypeStrings)) {
    return "Invalid";
  } else {
    return PatTypeStrings[PatType];
  }
}
 
BOOLEAN
OcIsPatSupported (
  VOID
  )
{
  CPUID_VERSION_INFO_EDX  Edx;
 
  //
  // Check CPUID(1).EDX[16] for PAT capability
  //
  AsmCpuid (CPUID_VERSION_INFO, NULL, NULL, NULL, &Edx.Uint32);
 
  return (Edx.Bits.PAT != 0);
}
 
PAGE_MAP_AND_DIRECTORY_POINTER  *
OcGetCurrentPageTable (
  OUT UINTN  *Flags  OPTIONAL
  )
{
  PAGE_MAP_AND_DIRECTORY_POINTER  *PageTable;
  UINTN                           Cr3;
 
  Cr3 = AsmReadCr3 ();
 
  PageTable = (PAGE_MAP_AND_DIRECTORY_POINTER *)(UINTN)(Cr3 & CR3_ADDR_MASK);
 
  if (Flags != NULL) {
    *Flags = Cr3 & (CR3_FLAG_PWT | CR3_FLAG_PCD);
  }
 
  return PageTable;
}
 
STATIC
BOOLEAN
DisablePageTableWriteProtection (
  VOID
  )
{
  UINTN  Cr0;
 
  Cr0 = AsmReadCr0 ();
 
  if ((Cr0 & CR0_WP) != 0) {
    AsmWriteCr0 (Cr0 & ~CR0_WP);
    return TRUE;
  }
 
  return FALSE;
}
 
STATIC
VOID
EnablePageTableWriteProtection (
  VOID
  )
{
  UINTN  Cr0;
 
  Cr0 = AsmReadCr0 ();
  AsmWriteCr0 (Cr0 | CR0_WP);
}
 
EFI_STATUS
OcGetPhysicalAddress (
  IN  PAGE_MAP_AND_DIRECTORY_POINTER  *PageTable  OPTIONAL,
  IN  EFI_VIRTUAL_ADDRESS             VirtualAddr,
  OUT EFI_PHYSICAL_ADDRESS            *PhysicalAddr
  )
{
  return OcGetSetPageTableInfoForAddress (
           PageTable,
           VirtualAddr,
           PhysicalAddr,
           NULL,
           NULL,
           NULL,
           FALSE
           );
}
 
EFI_STATUS
OcSetPatIndexForAddressRange (
  IN  PAGE_MAP_AND_DIRECTORY_POINTER  *PageTable  OPTIONAL,
  IN  EFI_VIRTUAL_ADDRESS             VirtualAddr,
  IN  UINT64                          Length,
  IN  PAT_INDEX                       *PatIndex
  )
{
  EFI_STATUS            Status;
  EFI_VIRTUAL_ADDRESS   EndAddr;
  EFI_PHYSICAL_ADDRESS  PhysicalAddr;
  UINT8                 Level;
 
  if (BaseOverflowAddU64 (VirtualAddr, Length, &EndAddr)) {
    return EFI_INVALID_PARAMETER;
  }
 
  do {
    Status = OcGetSetPageTableInfoForAddress (
               PageTable,
               VirtualAddr,
               &PhysicalAddr,
               &Level,
               NULL,
               PatIndex,
               TRUE
               );
 
    if (EFI_ERROR (Status)) {
      return Status;
    }
 
    if (VirtualAddr != PhysicalAddr) {
      return EFI_UNSUPPORTED;
    }
 
    switch (Level) {
      case 1:
        VirtualAddr += SIZE_1GB;
        break;
 
      case 2:
        VirtualAddr += SIZE_2MB;
        break;
 
      case 4:
        VirtualAddr += SIZE_4KB;
        break;
 
      default:
        return EFI_UNSUPPORTED;
    }
  } while (VirtualAddr < EndAddr);
 
  return EFI_SUCCESS;
}
 
EFI_STATUS
OcGetSetPageTableInfoForAddress (
  IN      PAGE_MAP_AND_DIRECTORY_POINTER  *PageTable    OPTIONAL,
  IN      EFI_VIRTUAL_ADDRESS             VirtualAddr,
  OUT     EFI_PHYSICAL_ADDRESS            *PhysicalAddr OPTIONAL,
  OUT     UINT8                           *Level        OPTIONAL,
  OUT     UINT64                          *Bits         OPTIONAL,
  IN OUT  PAT_INDEX                       *PatIndex     OPTIONAL,
  IN      BOOLEAN                         SetPat
  )
{
  EFI_PHYSICAL_ADDRESS            Start;
  VIRTUAL_ADDR                    VA;
  VIRTUAL_ADDR                    VAStart;
  VIRTUAL_ADDR                    VAEnd;
  PAGE_MAP_AND_DIRECTORY_POINTER  *PML4;
  PAGE_MAP_AND_DIRECTORY_POINTER  *PDPE;
  PAGE_MAP_AND_DIRECTORY_POINTER  *PDE;
  PAGE_TABLE_4K_ENTRY             *PTE4K;
  PAGE_TABLE_2M_ENTRY             *PTE2M;
  PAGE_TABLE_1G_ENTRY             *PTE1G;
 
  //
  // Ensure unused bits are zero.
  //
  if (!SetPat && (PatIndex != NULL)) {
    PatIndex->Index = 0;
  }
 
  if (PageTable == NULL) {
    PageTable = OcGetCurrentPageTable (NULL);
  }
 
  VA.Uint64 = (UINT64)VirtualAddr;
 
  //
  // PML4
  //
  PML4                    = PageTable;
  PML4                   += VA.Pg4K.PML4Offset;
  VAStart.Uint64          = 0;
  VAStart.Pg4K.PML4Offset = VA.Pg4K.PML4Offset;
  VA_FIX_SIGN_EXTEND (VAStart);
  VAEnd.Uint64          = ~(UINT64)0;
  VAEnd.Pg4K.PML4Offset = VA.Pg4K.PML4Offset;
  VA_FIX_SIGN_EXTEND (VAEnd);
 
  if (!PML4->Bits.Present) {
    return EFI_NO_MAPPING;
  }
 
  //
  // PDPE
  //
  PDPE                   = (PAGE_MAP_AND_DIRECTORY_POINTER *)(UINTN)(PML4->Uint64 & PAGING_4K_ADDRESS_MASK_64);
  PDPE                  += VA.Pg4K.PDPOffset;
  VAStart.Pg4K.PDPOffset = VA.Pg4K.PDPOffset;
  VAEnd.Pg4K.PDPOffset   = VA.Pg4K.PDPOffset;
 
  if (!PDPE->Bits.Present) {
    return EFI_NO_MAPPING;
  }
 
  if (PDPE->Bits.MustBeZero & 0x1) {
    //
    // 1GB PDPE
    //
    PTE1G = (PAGE_TABLE_1G_ENTRY *)PDPE;
    Start = PTE1G->Uint64 & PAGING_1G_ADDRESS_MASK_64;
 
    if (PhysicalAddr != NULL) {
      *PhysicalAddr = Start + VA.Pg1G.PhysPgOffset;
    }
 
    if (Level != NULL) {
      *Level = 1;
    }
 
    if (PatIndex != NULL) {
      if (SetPat) {
        PTE1G->Bits.WriteThrough  = PatIndex->Bits.WriteThrough;
        PTE1G->Bits.CacheDisabled = PatIndex->Bits.CacheDisabled;
        PTE1G->Bits.PAT           = PatIndex->Bits.PAT;
      } else {
        PatIndex->Bits.WriteThrough  = (UINT8)PTE1G->Bits.WriteThrough;
        PatIndex->Bits.CacheDisabled = (UINT8)PTE1G->Bits.CacheDisabled;
        PatIndex->Bits.PAT           = (UINT8)PTE1G->Bits.PAT;
      }
    }
 
    if (Bits != NULL) {
      *Bits = PTE1G->Uint64;
    }
 
    return EFI_SUCCESS;
  }
 
  //
  // PDE
  //
  PDE                   = (PAGE_MAP_AND_DIRECTORY_POINTER *)(UINTN)(PDPE->Uint64 & PAGING_4K_ADDRESS_MASK_64);
  PDE                  += VA.Pg4K.PDOffset;
  VAStart.Pg4K.PDOffset = VA.Pg4K.PDOffset;
  VAEnd.Pg4K.PDOffset   = VA.Pg4K.PDOffset;
 
  if (!PDE->Bits.Present) {
    return EFI_NO_MAPPING;
  }
 
  if (PDE->Bits.MustBeZero & 0x1) {
    //
    // 2MB PDE
    //
    PTE2M = (PAGE_TABLE_2M_ENTRY *)PDE;
    Start = PTE2M->Uint64 & PAGING_2M_ADDRESS_MASK_64;
 
    if (PhysicalAddr != NULL) {
      *PhysicalAddr = Start + VA.Pg2M.PhysPgOffset;
    }
 
    if (Level != NULL) {
      *Level = 2;
    }
 
    if (PatIndex != NULL) {
      if (SetPat) {
        PTE2M->Bits.WriteThrough  = PatIndex->Bits.WriteThrough;
        PTE2M->Bits.CacheDisabled = PatIndex->Bits.CacheDisabled;
        PTE2M->Bits.PAT           = PatIndex->Bits.PAT;
      } else {
        PatIndex->Bits.WriteThrough  = (UINT8)PTE2M->Bits.WriteThrough;
        PatIndex->Bits.CacheDisabled = (UINT8)PTE2M->Bits.CacheDisabled;
        PatIndex->Bits.PAT           = (UINT8)PTE2M->Bits.PAT;
      }
    }
 
    if (Bits != NULL) {
      *Bits = PTE2M->Uint64;
    }
 
    return EFI_SUCCESS;
  }
 
  //
  // PTE
  //
  PTE4K                 = (PAGE_TABLE_4K_ENTRY *)(UINTN)(PDE->Uint64 & PAGING_4K_ADDRESS_MASK_64);
  PTE4K                += VA.Pg4K.PTOffset;
  VAStart.Pg4K.PTOffset = VA.Pg4K.PTOffset;
  VAEnd.Pg4K.PTOffset   = VA.Pg4K.PTOffset;
 
  if (!PTE4K->Bits.Present) {
    return EFI_NO_MAPPING;
  }
 
  Start = PTE4K->Uint64 & PAGING_4K_ADDRESS_MASK_64;
 
  if (PhysicalAddr != NULL) {
    *PhysicalAddr = Start + VA.Pg4K.PhysPgOffset;
  }
 
  if (Level != NULL) {
    *Level = 4;
  }
 
  if (PatIndex != NULL) {
    if (SetPat) {
      PTE4K->Bits.WriteThrough  = PatIndex->Bits.WriteThrough;
      PTE4K->Bits.CacheDisabled = PatIndex->Bits.CacheDisabled;
      PTE4K->Bits.PAT           = PatIndex->Bits.PAT;
    } else {
      PatIndex->Bits.WriteThrough  = (UINT8)PTE4K->Bits.WriteThrough;
      PatIndex->Bits.CacheDisabled = (UINT8)PTE4K->Bits.CacheDisabled;
      PatIndex->Bits.PAT           = (UINT8)PTE4K->Bits.PAT;
    }
  }
 
  if (Bits != NULL) {
    *Bits = PTE4K->Uint64;
  }
 
  return EFI_SUCCESS;
}
 
//
// MTRR types would ideally be passed as MTRR_MEMORY_CACHE_TYPE, but this
// requires Library/MtrrLib.h in the .inf and .c files of everything which
// consumes OcMemoryLib.
//
EFI_STATUS
OcModifyMtrrRange (
  IN  EFI_PHYSICAL_ADDRESS  Address,
  IN  UINT8                 MtrrType,
  OUT UINT64                *Length   OPTIONAL
  )
{
  EFI_PHYSICAL_ADDRESS  Walker;
  UINT8                 OriginalMtrrType;
 
  if (Length != NULL) {
    *Length = 0;
  }
 
  if (!IsMtrrSupported ()) {
    return EFI_UNSUPPORTED;
  }
 
  Walker           = Address;
  OriginalMtrrType = MtrrGetMemoryAttribute (Walker);
  do {
    Walker += SIZE_2MB;
  } while (OriginalMtrrType == MtrrGetMemoryAttribute (Walker));
 
  if (Length != NULL) {
    *Length = Walker - Address;
  }
 
  return MtrrSetMemoryAttribute (Address, Walker - Address, MtrrType);
}
 
EFI_STATUS
VmAllocateMemoryPool (
  OUT OC_VMEM_CONTEXT     *Context,
  IN  UINTN               NumPages,
  IN  EFI_GET_MEMORY_MAP  GetMemoryMap  OPTIONAL
  )
{
  EFI_STATUS            Status;
  EFI_PHYSICAL_ADDRESS  Addr;
 
  Addr   = BASE_4GB;
  Status = OcAllocatePagesFromTop (
             EfiBootServicesData,
             NumPages,
             &Addr,
             GetMemoryMap,
             NULL,
             NULL
             );
 
  if (!EFI_ERROR (Status)) {
    Context->MemoryPool = (UINT8 *)(UINTN)Addr;
    Context->FreePages  = NumPages;
  }
 
  return Status;
}
 
VOID *
VmAllocatePages (
  IN OUT OC_VMEM_CONTEXT  *Context,
  IN     UINTN            NumPages
  )
{
  VOID  *AllocatedPages;
 
  AllocatedPages = NULL;
 
  if (Context->FreePages >= NumPages) {
    AllocatedPages       = Context->MemoryPool;
    Context->MemoryPool += EFI_PAGES_TO_SIZE (NumPages);
    Context->FreePages  -= NumPages;
  }
 
  return AllocatedPages;
}
 
EFI_STATUS
VmMapVirtualPage (
  IN OUT OC_VMEM_CONTEXT                 *Context,
  IN OUT PAGE_MAP_AND_DIRECTORY_POINTER  *PageTable  OPTIONAL,
  IN     EFI_VIRTUAL_ADDRESS             VirtualAddr,
  IN     EFI_PHYSICAL_ADDRESS            PhysicalAddr
  )
{
  EFI_PHYSICAL_ADDRESS            Start;
  VIRTUAL_ADDR                    VA;
  VIRTUAL_ADDR                    VAStart;
  VIRTUAL_ADDR                    VAEnd;
  PAGE_MAP_AND_DIRECTORY_POINTER  *PML4;
  PAGE_MAP_AND_DIRECTORY_POINTER  *PDPE;
  PAGE_MAP_AND_DIRECTORY_POINTER  *PDE;
  PAGE_TABLE_4K_ENTRY             *PTE4K;
  PAGE_TABLE_4K_ENTRY             *PTE4KTmp;
  PAGE_TABLE_2M_ENTRY             *PTE2M;
  PAGE_TABLE_1G_ENTRY             *PTE1G;
  UINTN                           Index;
  BOOLEAN                         WriteProtected;
 
  if (PageTable == NULL) {
    PageTable = OcGetCurrentPageTable (NULL);
  }
 
  WriteProtected = DisablePageTableWriteProtection ();
 
  VA.Uint64 = (UINT64)VirtualAddr;
 
  //
  // PML4
  //
  PML4  = PageTable;
  PML4 += VA.Pg4K.PML4Offset;
 
  //
  // There is a problem if our PML4 points to the same table as first PML4 entry
  // since we may mess the mapping of first virtual region (happens in VBox and probably DUET).
  // Check for this on first call and if true, just clear our PML4 - we'll rebuild at a later step.
  //
  if (  (PML4 != PageTable) && PML4->Bits.Present
     && (PageTable->Bits.PageTableBaseAddress == PML4->Bits.PageTableBaseAddress))
  {
    PML4->Uint64 = 0;
  }
 
  VAStart.Uint64          = 0;
  VAStart.Pg4K.PML4Offset = VA.Pg4K.PML4Offset;
  VA_FIX_SIGN_EXTEND (VAStart);
  VAEnd.Uint64          = ~(UINT64)0;
  VAEnd.Pg4K.PML4Offset = VA.Pg4K.PML4Offset;
  VA_FIX_SIGN_EXTEND (VAEnd);
 
  if (!PML4->Bits.Present) {
    PDPE = (PAGE_MAP_AND_DIRECTORY_POINTER *)VmAllocatePages (Context, 1);
 
    if (PDPE == NULL) {
      if (WriteProtected) {
        EnablePageTableWriteProtection ();
      }
 
      return EFI_NO_MAPPING;
    }
 
    ZeroMem (PDPE, EFI_PAGE_SIZE);
 
    //
    // Init this whole 512 GB region with 512 1GB entry pages to map
    // the first 512 GB physical space.
    //
    PTE1G = (PAGE_TABLE_1G_ENTRY *)PDPE;
    Start = 0;
    for (Index = 0; Index < 512; ++Index) {
      PTE1G->Uint64         = Start & PAGING_1G_ADDRESS_MASK_64;
      PTE1G->Bits.ReadWrite = 1;
      PTE1G->Bits.Present   = 1;
      PTE1G->Bits.MustBe1   = 1;
      PTE1G++;
      Start += BASE_1GB;
    }
 
    //
    // Put it to PML4.
    //
    PML4->Uint64         = ((UINT64)(UINTN)PDPE) & PAGING_4K_ADDRESS_MASK_64;
    PML4->Bits.ReadWrite = 1;
    PML4->Bits.Present   = 1;
  }
 
  //
  // PDPE
  //
  PDPE                   = (PAGE_MAP_AND_DIRECTORY_POINTER *)(UINTN)(PML4->Uint64 & PAGING_4K_ADDRESS_MASK_64);
  PDPE                  += VA.Pg4K.PDPOffset;
  VAStart.Pg4K.PDPOffset = VA.Pg4K.PDPOffset;
  VAEnd.Pg4K.PDPOffset   = VA.Pg4K.PDPOffset;
 
  if (!PDPE->Bits.Present || (PDPE->Bits.MustBeZero & 0x1)) {
    PDE = (PAGE_MAP_AND_DIRECTORY_POINTER *)VmAllocatePages (Context, 1);
 
    if (PDE == NULL) {
      if (WriteProtected) {
        EnablePageTableWriteProtection ();
      }
 
      return EFI_NO_MAPPING;
    }
 
    ZeroMem (PDE, EFI_PAGE_SIZE);
 
    if (PDPE->Bits.MustBeZero & 0x1) {
      //
      // This is 1 GB page. Init new PDE array to get the same
      // mapping but with 2MB pages.
      //
      PTE2M = (PAGE_TABLE_2M_ENTRY *)PDE;
      Start = PDPE->Uint64 & PAGING_1G_ADDRESS_MASK_64;
 
      for (Index = 0; Index < 512; ++Index) {
        PTE2M->Uint64         = Start & PAGING_2M_ADDRESS_MASK_64;
        PTE2M->Bits.ReadWrite = 1;
        PTE2M->Bits.Present   = 1;
        PTE2M->Bits.MustBe1   = 1;
        PTE2M++;
        Start += BASE_2MB;
      }
    }
 
    //
    // Put it to PDPE.
    //
    PDPE->Uint64         = ((UINT64)(UINTN)PDE) & PAGING_4K_ADDRESS_MASK_64;
    PDPE->Bits.ReadWrite = 1;
    PDPE->Bits.Present   = 1;
  }
 
  //
  // PDE
  //
  PDE                   = (PAGE_MAP_AND_DIRECTORY_POINTER *)(UINTN)(PDPE->Uint64 & PAGING_4K_ADDRESS_MASK_64);
  PDE                  += VA.Pg4K.PDOffset;
  VAStart.Pg4K.PDOffset = VA.Pg4K.PDOffset;
  VAEnd.Pg4K.PDOffset   = VA.Pg4K.PDOffset;
 
  if (!PDE->Bits.Present || (PDE->Bits.MustBeZero & 0x1)) {
    PTE4K = (PAGE_TABLE_4K_ENTRY *)VmAllocatePages (Context, 1);
 
    if (PTE4K == NULL) {
      if (WriteProtected) {
        EnablePageTableWriteProtection ();
      }
 
      return EFI_NO_MAPPING;
    }
 
    ZeroMem (PTE4K, EFI_PAGE_SIZE);
 
    if (PDE->Bits.MustBeZero & 0x1) {
      //
      // This is 2 MB page. Init new PTE array to get the same
      // mapping but with 4KB pages.
      //
      PTE4KTmp = (PAGE_TABLE_4K_ENTRY *)PTE4K;
      Start    = PDE->Uint64 & PAGING_2M_ADDRESS_MASK_64;
 
      for (Index = 0; Index < 512; ++Index) {
        PTE4KTmp->Uint64         = Start & PAGING_4K_ADDRESS_MASK_64;
        PTE4KTmp->Bits.ReadWrite = 1;
        PTE4KTmp->Bits.Present   = 1;
        PTE4KTmp++;
        Start += BASE_4KB;
      }
    }
 
    //
    // Put it to PDE.
    //
    PDE->Uint64         = ((UINT64)(UINTN)PTE4K) & PAGING_4K_ADDRESS_MASK_64;
    PDE->Bits.ReadWrite = 1;
    PDE->Bits.Present   = 1;
  }
 
  //
  // PTE
  //
  PTE4K                 = (PAGE_TABLE_4K_ENTRY *)(UINTN)(PDE->Uint64 & PAGING_4K_ADDRESS_MASK_64);
  PTE4K                += VA.Pg4K.PTOffset;
  VAStart.Pg4K.PTOffset = VA.Pg4K.PTOffset;
  VAEnd.Pg4K.PTOffset   = VA.Pg4K.PTOffset;
 
  //
  // Put it to PTE.
  //
  PTE4K->Uint64         = ((UINT64)PhysicalAddr) & PAGING_4K_ADDRESS_MASK_64;
  PTE4K->Bits.ReadWrite = 1;
  PTE4K->Bits.Present   = 1;
 
  if (WriteProtected) {
    EnablePageTableWriteProtection ();
  }
 
  return EFI_SUCCESS;
}
 
EFI_STATUS
VmMapVirtualPages (
  IN OUT OC_VMEM_CONTEXT                 *Context,
  IN OUT PAGE_MAP_AND_DIRECTORY_POINTER  *PageTable  OPTIONAL,
  IN     EFI_VIRTUAL_ADDRESS             VirtualAddr,
  IN     UINT64                          NumPages,
  IN     EFI_PHYSICAL_ADDRESS            PhysicalAddr
  )
{
  EFI_STATUS  Status;
 
  if (PageTable == NULL) {
    PageTable = OcGetCurrentPageTable (NULL);
  }
 
  Status = EFI_SUCCESS;
 
  while (NumPages > 0 && !EFI_ERROR (Status)) {
    Status = VmMapVirtualPage (
               Context,
               PageTable,
               VirtualAddr,
               PhysicalAddr
               );
 
    VirtualAddr  += EFI_PAGE_SIZE;
    PhysicalAddr += EFI_PAGE_SIZE;
    NumPages--;
  }
 
  return Status;
}
 
VOID
VmFlushCaches (
  VOID
  )
{
  //
  // Simply reload CR3 register.
  //
  AsmWriteCr3 (AsmReadCr3 ());
}