mirror of
https://github.com/jart/cosmopolitan.git
synced 2024-05-17 02:52:40 +00:00
2ab9e9f7fd
- Introduce portable sched_getcpu() api - Support GCC's __target_clones__ feature - Make fma() go faster on x86 in default mode - Remove some asan checks from core libraries - WinMain() now ensures $HOME and $USER are defined
358 lines
15 KiB
C
358 lines
15 KiB
C
/*-*- mode:c;indent-tabs-mode:nil;c-basic-offset:2;tab-width:8;coding:utf-8 -*-│
|
||
│ vi: set et ft=c ts=2 sts=2 sw=2 fenc=utf-8 :vi │
|
||
╞══════════════════════════════════════════════════════════════════════════════╡
|
||
│ Copyright 2020 Justine Alexandra Roberts Tunney │
|
||
│ │
|
||
│ Permission to use, copy, modify, and/or distribute this software for │
|
||
│ any purpose with or without fee is hereby granted, provided that the │
|
||
│ above copyright notice and this permission notice appear in all copies. │
|
||
│ │
|
||
│ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL │
|
||
│ WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED │
|
||
│ WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE │
|
||
│ AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL │
|
||
│ DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR │
|
||
│ PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER │
|
||
│ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR │
|
||
│ PERFORMANCE OF THIS SOFTWARE. │
|
||
╠──────────────────────────────────────────────────────────────────────────────╣
|
||
│░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░│
|
||
│░░░░░░░█▀█░█▀█░▀█▀░█░█░█▀█░█░░░█░░░█░█░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░│
|
||
│░░░░░░░█▀█░█░▄░░█░░█░█░█▀█░█░░░█░░░▀█▀░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░│
|
||
│░░░░░░░▀░▀░▀▀▀░░▀░░▀▀▀░▀░▀░▀▀▀░▀▀▀░░▀░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░│
|
||
│░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░│
|
||
│░░░░░░░█▀█░█▀█░█▀█░▀█▀░█▀█░█▀█░█░░░█▀▀░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░│
|
||
│░░░░░░░█▀▀░█ █░██▀░░█░░█▀█░█▀█░█░░░█▀▀░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░│
|
||
│░░░░░░░▀░░░▀▀▀░▀░▀░░▀░░▀░▀░▀▀▀░▀▀▀░▀▀▀░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░│
|
||
│░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░│
|
||
│░░░░░░░█▀▀░█░█░█▀▀░█▀█░█░█░▀█▀░█▀█░█▀█░█░░█▀▀░░░░░░░░░░░░░░░░░░░░░░░░▄▄░░░▐█░░│
|
||
│░░░░░░░█▀▀░▄▀▄░█▀▀░█░▄░█░█░░█░░█▀█░█▀█░█░░█▀▀░░░░░░░░░░░░▄▄▄░░░▄██▄░░█▀░░░█░▄░│
|
||
│░░░░░░░▀▀▀░▀░▀░▀▀▀░▀▀▀░▀▀▀░░▀░░▀░▀░▀▀▀░▀▀░▀▀▀░░░░░░░░░░▄██▀█▌░██▄▄░░▐█▀▄░▐█▀░░│
|
||
│░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░▐█▀▀▌░░░▄▀▌░▌░█░▌░░▌░▌░░│
|
||
╠──────────────────────────────────────────────────────▌▀▄─▐──▀▄─▐▄─▐▄▐▄─▐▄─▐▄─│
|
||
│ αcτµαlly pδrταblε εxεcµταblε § no-frills virtual memory management │
|
||
╚─────────────────────────────────────────────────────────────────────────────*/
|
||
#include "ape/relocations.h"
|
||
#include "libc/assert.h"
|
||
#include "libc/elf/def.h"
|
||
#include "libc/elf/struct/phdr.h"
|
||
#include "libc/macros.internal.h"
|
||
#include "libc/nexgen32e/uart.internal.h"
|
||
#include "libc/runtime/e820.internal.h"
|
||
#include "libc/runtime/metalprintf.internal.h"
|
||
#include "libc/runtime/pc.internal.h"
|
||
#include "libc/runtime/runtime.h"
|
||
#ifdef __x86_64__
|
||
|
||
#define INVERT(x) (BANE + PHYSICAL((uintptr_t)(x)))
|
||
#define NOPAGE ((uint64_t) - 1)
|
||
|
||
#define APE_STACK_VADDR \
|
||
({ \
|
||
int64_t vAddr; \
|
||
__asm__(".weak\tape_stack_vaddr\n\t" \
|
||
"movabs\t$ape_stack_vaddr,%0" \
|
||
: "=r"(vAddr)); \
|
||
vAddr; \
|
||
})
|
||
|
||
struct ReclaimedPage {
|
||
uint64_t next;
|
||
};
|
||
|
||
/**
|
||
* @internal
|
||
* Allocates new page of physical memory.
|
||
*/
|
||
texthead uint64_t __new_page(struct mman *mm) {
|
||
uint64_t p = mm->frp;
|
||
if (p != NOPAGE) {
|
||
uint64_t q;
|
||
struct ReclaimedPage *rp = (struct ReclaimedPage *)(BANE + p);
|
||
unassert(p == (p & PAGE_TA));
|
||
q = rp->next;
|
||
unassert(q == (q & PAGE_TA) || q == NOPAGE);
|
||
mm->frp = q;
|
||
return p;
|
||
}
|
||
if (mm->pdpi == mm->e820n) {
|
||
return 0;
|
||
}
|
||
while (mm->pdp >= mm->e820[mm->pdpi].addr + mm->e820[mm->pdpi].size) {
|
||
if (++mm->pdpi == mm->e820n) return 0;
|
||
mm->pdp = MAX(mm->pdp, mm->e820[mm->pdpi].addr);
|
||
}
|
||
p = mm->pdp;
|
||
mm->pdp += 4096;
|
||
return p;
|
||
}
|
||
|
||
/**
|
||
* @internal
|
||
* Returns pointer to page table entry for page at virtual address.
|
||
* Additional page tables are allocated if needed as a side-effect.
|
||
*/
|
||
textreal uint64_t *__get_virtual(struct mman *mm, uint64_t *t, int64_t vaddr,
|
||
bool maketables) {
|
||
uint64_t *e, p;
|
||
unsigned char h;
|
||
for (h = 39;; h -= 9) {
|
||
e = t + ((vaddr >> h) & 511);
|
||
if (h == 12) return e;
|
||
if (!(*e & (PAGE_V | PAGE_RSRV))) {
|
||
if (!maketables) return NULL;
|
||
if (!(p = __new_page(mm))) return NULL;
|
||
__clear_page(BANE + p);
|
||
*e = p | PAGE_V | PAGE_RW;
|
||
}
|
||
t = (uint64_t *)(BANE + (*e & PAGE_TA));
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @internal
|
||
* Sorts, rounds, and filters BIOS memory map.
|
||
*/
|
||
static textreal void __normalize_e820(struct mman *mm, uint64_t top) {
|
||
uint64_t a, b;
|
||
uint64_t x, y;
|
||
unsigned i, j, n;
|
||
for (n = i = 0; mm->e820[i].size; ++i) {
|
||
mm->e820[n] = mm->e820[i];
|
||
x = mm->e820[n].addr;
|
||
y = mm->e820[n].addr + mm->e820[n].size;
|
||
a = ROUNDUP(x, 4096);
|
||
b = ROUNDDOWN(y, 4096);
|
||
if (b > a && mm->e820[i].type == kMemoryUsable) {
|
||
b -= a;
|
||
mm->e820[n].addr = a;
|
||
mm->e820[n].size = b;
|
||
++n;
|
||
}
|
||
}
|
||
for (i = 1; i < n; ++i) {
|
||
for (j = i; j > 0 && mm->e820[i].addr < mm->e820[j - 1].addr; --j) {
|
||
mm->e820[j] = mm->e820[j - 1];
|
||
}
|
||
mm->e820[j] = mm->e820[i];
|
||
}
|
||
top = ROUNDUP(top, 4096);
|
||
mm->pdp = MAX(top, mm->e820[0].addr);
|
||
mm->pdpi = 0;
|
||
mm->e820n = n;
|
||
mm->frp = NOPAGE;
|
||
}
|
||
|
||
/**
|
||
* @internal
|
||
* Identity maps an area of physical memory to its negative address.
|
||
*/
|
||
textreal uint64_t *__invert_memory_area(struct mman *mm, uint64_t *pml4t,
|
||
uint64_t ps, uint64_t size,
|
||
uint64_t pte_flags) {
|
||
uint64_t pe = ps + size, p, *m = NULL;
|
||
ps = ROUNDDOWN(ps, 4096);
|
||
pe = ROUNDUP(pe, 4096);
|
||
for (p = ps; p != pe; p += 4096) {
|
||
m = __get_virtual(mm, pml4t, BANE + p, true);
|
||
if (m && !(*m & (PAGE_V | PAGE_RSRV))) {
|
||
*m = p | PAGE_V | PAGE_RSRV | pte_flags;
|
||
}
|
||
}
|
||
return m;
|
||
}
|
||
|
||
/**
|
||
* @internal
|
||
* Increments the reference count for a page of physical memory.
|
||
*/
|
||
void __ref_page(struct mman *mm, uint64_t *pml4t, uint64_t p) {
|
||
uint64_t *m, e;
|
||
m = __invert_memory_area(mm, pml4t, p, 4096, PAGE_RW | PAGE_XD);
|
||
if (m) {
|
||
e = *m;
|
||
if ((e & PAGE_REFC) != PAGE_REFC) {
|
||
e += PAGE_1REF;
|
||
*m = e;
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @internal
|
||
* Increments the reference counts for an area of physical memory.
|
||
*/
|
||
void __ref_pages(struct mman *mm, uint64_t *pml4t, uint64_t ps, uint64_t size) {
|
||
uint64_t p = ROUNDDOWN(ps, 4096), e = ROUNDUP(ps + size, 4096);
|
||
while (p != e) {
|
||
__ref_page(mm, pml4t, p);
|
||
p += 4096;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @internal
|
||
* Reclaims a page of physical memory for later use.
|
||
*/
|
||
static void __reclaim_page(struct mman *mm, uint64_t p) {
|
||
struct ReclaimedPage *rp = (struct ReclaimedPage *)(BANE + p);
|
||
unassert(p == (p & PAGE_TA));
|
||
rp->next = mm->frp;
|
||
mm->frp = p;
|
||
}
|
||
|
||
/**
|
||
* @internal
|
||
* Decrements the reference count for a page of physical memory. Frees the
|
||
* page if there are no virtual addresses (excluding the negative space)
|
||
* referring to it.
|
||
*/
|
||
void __unref_page(struct mman *mm, uint64_t *pml4t, uint64_t p) {
|
||
uint64_t *m, e;
|
||
m = __invert_memory_area(mm, pml4t, p, 4096, PAGE_RW | PAGE_XD);
|
||
if (m) {
|
||
e = *m;
|
||
if ((e & PAGE_REFC) != PAGE_REFC) {
|
||
e -= PAGE_1REF;
|
||
*m = e;
|
||
if ((e & PAGE_REFC) == 0) __reclaim_page(mm, p);
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @internal
|
||
* Identity maps all usable physical memory to its negative address.
|
||
*/
|
||
static textreal void __invert_memory(struct mman *mm, uint64_t *pml4t) {
|
||
uint64_t i;
|
||
for (i = 0; i < mm->e820n; ++i) {
|
||
uint64_t ps = mm->e820[i].addr, size = mm->e820[i].size;
|
||
/* ape/ape.S has already mapped the first 2 MiB of physical memory. */
|
||
if (ps < 0x200000 && ps + size <= 0x200000) continue;
|
||
__invert_memory_area(mm, pml4t, ps, size, PAGE_RW | PAGE_XD);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* @internal
|
||
* Exports information about the offset of a field within a structure type,
|
||
* so that assembly language routines can use it. This macro can be invoked
|
||
* from inside a function whose code is known to be emitted.
|
||
*/
|
||
#define export_offsetof(type, member) \
|
||
do { \
|
||
asm volatile(".globl \"" #type "::" #member "\"\n\t" \
|
||
".set \"" #type "::" #member "\",%c0" \
|
||
: /* no outputs */ \
|
||
: "i"(offsetof(type, member))); \
|
||
} while (0)
|
||
|
||
textreal void __setup_mman(struct mman *mm, uint64_t *pml4t, uint64_t top) {
|
||
export_offsetof(struct mman, pc_drive_base_table);
|
||
export_offsetof(struct mman, pc_drive_last_sector);
|
||
export_offsetof(struct mman, pc_drive_last_head);
|
||
export_offsetof(struct mman, pc_drive);
|
||
export_offsetof(struct mman, e820);
|
||
export_offsetof(struct mman, e820_end);
|
||
export_offsetof(struct mman, bad_idt);
|
||
export_offsetof(struct mman, pc_drive_next_sector);
|
||
export_offsetof(struct mman, pc_drive_next_cylinder);
|
||
export_offsetof(struct mman, pc_drive_next_head);
|
||
export_offsetof(struct mman, pc_video_type);
|
||
export_offsetof(struct mman, pc_video_stride);
|
||
export_offsetof(struct mman, pc_video_width);
|
||
export_offsetof(struct mman, pc_video_height);
|
||
export_offsetof(struct mman, pc_video_framebuffer);
|
||
export_offsetof(struct mman, pc_video_framebuffer_size);
|
||
export_offsetof(struct mman, pc_video_curs_info);
|
||
export_offsetof(struct mman, pc_video_char_height);
|
||
__normalize_e820(mm, top);
|
||
__invert_memory(mm, pml4t);
|
||
}
|
||
|
||
static textreal uint64_t __map_phdr(struct mman *mm, uint64_t *pml4t,
|
||
uint64_t b, uint64_t m,
|
||
struct Elf64_Phdr *p) {
|
||
uint64_t i, f, v;
|
||
if (p->p_type != PT_LOAD) return m;
|
||
f = PAGE_RSRV | PAGE_U;
|
||
if (p->p_flags & PF_W)
|
||
f |= PAGE_V | PAGE_RW;
|
||
else if (p->p_flags & (PF_R | PF_X))
|
||
f |= PAGE_V;
|
||
if (!(p->p_flags & PF_X)) f |= PAGE_XD;
|
||
for (i = 0; i < p->p_memsz; i += 4096) {
|
||
if (i < p->p_filesz) {
|
||
v = b + p->p_offset + i;
|
||
m = MAX(m, v);
|
||
} else {
|
||
v = __clear_page(BANE + __new_page(mm));
|
||
}
|
||
*__get_virtual(mm, pml4t, p->p_vaddr + i, true) = (v & PAGE_TA) | f;
|
||
__ref_page(mm, pml4t, v & PAGE_TA);
|
||
}
|
||
return m;
|
||
}
|
||
|
||
/**
|
||
* @internal
|
||
* Maps APE-defined ELF program headers into memory and clears BSS.
|
||
*/
|
||
textreal void __map_phdrs(struct mman *mm, uint64_t *pml4t, uint64_t b,
|
||
uint64_t top) {
|
||
uint64_t m;
|
||
struct Elf64_Phdr *p;
|
||
extern char ape_phdrs[] __attribute__((__weak__));
|
||
extern char ape_phdrs_end[] __attribute__((__weak__));
|
||
extern char ape_stack_pf[] __attribute__((__weak__));
|
||
extern char ape_stack_offset[] __attribute__((__weak__));
|
||
extern char ape_stack_filesz[] __attribute__((__weak__));
|
||
extern char ape_stack_memsz[] __attribute__((__weak__));
|
||
__setup_mman(mm, pml4t, top);
|
||
for (p = (struct Elf64_Phdr *)INVERT(ape_phdrs), m = 0;
|
||
p < (struct Elf64_Phdr *)INVERT(ape_phdrs_end); ++p) {
|
||
m = __map_phdr(mm, pml4t, b, m, p);
|
||
}
|
||
m = __map_phdr(mm, pml4t, b, m,
|
||
&(struct Elf64_Phdr){
|
||
.p_type = PT_LOAD,
|
||
.p_flags = (uintptr_t)ape_stack_pf,
|
||
.p_offset = (uintptr_t)ape_stack_offset,
|
||
.p_vaddr = APE_STACK_VADDR,
|
||
.p_filesz = (uintptr_t)ape_stack_filesz,
|
||
.p_memsz = (uintptr_t)ape_stack_memsz,
|
||
});
|
||
mm->pdp = MAX(mm->pdp, m);
|
||
}
|
||
|
||
/**
|
||
* @internal
|
||
* Reclaims memory pages which were used at boot time but which can now be
|
||
* made available for the application.
|
||
*/
|
||
textreal void __reclaim_boot_pages(struct mman *mm, uint64_t skip_start,
|
||
uint64_t skip_end) {
|
||
uint64_t p = mm->frp, q = IMAGE_BASE_REAL, i, n = mm->e820n, b, e;
|
||
for (i = 0; i < n; ++i) {
|
||
b = mm->e820[i].addr;
|
||
if (b >= IMAGE_BASE_PHYSICAL) break;
|
||
e = MIN(IMAGE_BASE_PHYSICAL, b + mm->e820[i].size);
|
||
q = MAX(IMAGE_BASE_REAL, b);
|
||
while (q < e) {
|
||
struct ReclaimedPage *rp;
|
||
if (q == skip_start) {
|
||
q = skip_end;
|
||
if (q >= e) break;
|
||
}
|
||
rp = (struct ReclaimedPage *)(BANE + q);
|
||
rp->next = p;
|
||
p = q;
|
||
q += 4096;
|
||
}
|
||
}
|
||
mm->frp = p;
|
||
}
|
||
|
||
#endif /* __x86_64__ */
|