From 2dd019ff0d3732974a1ba29c378740f792fc0af2 Mon Sep 17 00:00:00 2001 From: Nick Gasson Date: Tue, 28 Mar 2023 21:22:44 +0100 Subject: [PATCH] Create a single string table per LLVM translation unit --- src/jit/jit-core.c | 12 +- src/jit/jit-llvm.c | 108 +++++++------- src/jit/jit-pack.c | 346 +++++++++++++++++++++------------------------ src/jit/jit-priv.h | 13 +- src/rt/rt.h | 2 +- 5 files changed, 236 insertions(+), 245 deletions(-) diff --git a/src/jit/jit-core.c b/src/jit/jit-core.c index 46155a53..ed853ffd 100644 --- a/src/jit/jit-core.c +++ b/src/jit/jit-core.c @@ -74,13 +74,16 @@ typedef struct _aot_dll { typedef struct { reloc_kind_t kind; - void *ptr; + union { + uintptr_t off; + void *ptr; + }; } aot_reloc_t; // The code generator knows the layout of this struct typedef struct { jit_entry_fn_t entry; - ffi_spec_t spec; + const char *strtab; const uint8_t *debug; const uint8_t *cpool; aot_reloc_t relocs[0]; @@ -257,7 +260,8 @@ static jit_handle_t jit_lazy_compile_locked(jit_t *j, ident_t name) if (try[i] == NULL) continue; else if ((descr = ffi_find_symbol(try[i]->dll, tb_get(tb)))) { - jit_pack_put(try[i]->pack, name, descr->cpool, descr->debug); + jit_pack_put(try[i]->pack, name, descr->cpool, + descr->strtab, descr->debug); break; } } @@ -283,7 +287,7 @@ static jit_handle_t jit_lazy_compile_locked(jit_t *j, ident_t name) if (descr != NULL) { for (aot_reloc_t *r = descr->relocs; r->kind != RELOC_NULL; r++) { - const char *str = (char *)r->ptr; + const char *str = descr->strtab + r->off; if (r->kind == RELOC_FOREIGN) { const char *eptr = strchr(str, '\b'); if (eptr == NULL) diff --git a/src/jit/jit-llvm.c b/src/jit/jit-llvm.c index 7a58cec0..c0b70b1b 100644 --- a/src/jit/jit-llvm.c +++ b/src/jit/jit-llvm.c @@ -67,6 +67,7 @@ typedef enum { LLVM_ANCHOR, LLVM_TLAB, LLVM_AOT_RELOC, + LLVM_STRTAB, LLVM_LAST_TYPE } llvm_type_t; @@ -161,14 +162,14 @@ typedef struct _llvm_obj { LLVMBuilderRef builder; #if ENABLE_DWARF LLVMDIBuilderRef debuginfo; + LLVMMetadataRef debugcu; #endif LLVMTargetDataRef data_ref; LLVMTypeRef types[LLVM_LAST_TYPE]; LLVMValueRef fns[LLVM_LAST_FN]; LLVMTypeRef fntypes[LLVM_LAST_FN]; - LLVMMetadataRef debugcu; - shash_t *string_pool; - jit_pack_t *jitpack; + LLVMValueRef strtab; + pack_writer_t *pack_writer; } llvm_obj_t; typedef struct _cgen_block { @@ -344,6 +345,8 @@ static void llvm_register_types(llvm_obj_t *obj) obj->types[LLVM_PTR] = LLVMPointerType(obj->types[LLVM_INT8], 0); #endif + obj->types[LLVM_STRTAB] = LLVMArrayType(obj->types[LLVM_INT8], 0); + { LLVMTypeRef fields[] = { obj->types[LLVM_PTR], // Mspace object @@ -379,7 +382,7 @@ static void llvm_register_types(llvm_obj_t *obj) { LLVMTypeRef fields[] = { obj->types[LLVM_INT32], // Kind - obj->types[LLVM_PTR], // Data pointer + obj->types[LLVM_INTPTR], // Data pointer }; obj->types[LLVM_AOT_RELOC] = LLVMStructTypeInContext(obj->context, fields, ARRAY_LEN(fields), @@ -949,39 +952,14 @@ static LLVMValueRef llvm_call_fn(llvm_obj_t *obj, llvm_fn_t which, static LLVMValueRef llvm_const_string(llvm_obj_t *obj, const char *str) { - if (obj->string_pool == NULL) - obj->string_pool = shash_new(256); - - LLVMTypeRef array_type = NULL; - LLVMValueRef ref = shash_get(obj->string_pool, str); - if (ref == NULL) { - const size_t len = strlen(str); - LLVMValueRef init = - LLVMConstStringInContext(obj->context, str, len, false); - array_type = LLVMArrayType(obj->types[LLVM_INT8], len + 1); - ref = LLVMAddGlobal(obj->module, array_type, "const_string"); - LLVMSetGlobalConstant(ref, true); - LLVMSetInitializer(ref, init); - LLVMSetLinkage(ref, LLVMPrivateLinkage); - LLVMSetUnnamedAddr(ref, true); - - shash_put(obj->string_pool, str, ref); - } -#ifndef LLVM_HAS_OPAQUE_POINTERS - else - array_type = LLVMArrayType(obj->types[LLVM_INT8], strlen(str) + 1); -#endif + const unsigned off = pack_writer_get_string(obj->pack_writer, str); -#ifdef LLVM_HAS_OPAQUE_POINTERS - return ref; -#else LLVMValueRef indexes[] = { llvm_int32(obj, 0), - llvm_int32(obj, 0) + llvm_int32(obj, off) }; - return LLVMBuildGEP2(obj->builder, array_type, ref, indexes, - ARRAY_LEN(indexes), ""); -#endif + return LLVMBuildGEP2(obj->builder, obj->types[LLVM_STRTAB], obj->strtab, + indexes, ARRAY_LEN(indexes), "string"); } #if ENABLE_DWARF @@ -2473,11 +2451,9 @@ static LLVMValueRef cgen_debug_irbuf(llvm_obj_t *obj, jit_func_t *f) tb_istr(tb, f->name); tb_cat(tb, ".debug"); - jit_pack_encode(obj->jitpack, f->jit, f->handle); - - size_t size; - const uint8_t *buf = jit_pack_get(obj->jitpack, f->name, &size); - assert(buf != NULL); + size_t size = 0; + uint8_t *buf LOCAL = NULL; + pack_writer_emit(obj->pack_writer, f->jit, f->handle, &buf, &size); LLVMValueRef *data LOCAL = xmalloc_array(size, sizeof(LLVMValueRef)); for (size_t i = 0; i < size; i++) @@ -2496,6 +2472,12 @@ static LLVMValueRef cgen_debug_irbuf(llvm_obj_t *obj, jit_func_t *f) return global; } +static LLVMValueRef cgen_reloc_str(llvm_obj_t *obj, const char *str) +{ + const unsigned off = pack_writer_get_string(obj->pack_writer, str); + return llvm_intptr(obj, off); +} + static void cgen_aot_descr(llvm_obj_t *obj, cgen_func_t *func) { LLVMValueRef irbuf = cgen_debug_irbuf(obj, func->source); @@ -2513,7 +2495,7 @@ static void cgen_aot_descr(llvm_obj_t *obj, cgen_func_t *func) jit_func_t *f = jit_get_func(func->source->jit, ir->arg1.handle); const cgen_reloc_t r = { .kind = kind, - .str = llvm_const_string(obj, istr(f->name)), + .str = cgen_reloc_str(obj, istr(f->name)), .key = ir->arg1.handle, .nth = relocs.count, }; @@ -2529,11 +2511,10 @@ static void cgen_aot_descr(llvm_obj_t *obj, cgen_func_t *func) args[j].handle) == NULL) { ident_t name = jit_get_name(func->source->jit, args[j].handle); - LLVMValueRef str = llvm_const_string(obj, istr(name)); const cgen_reloc_t r = { .kind = RELOC_HANDLE, - .str = str, + .str = cgen_reloc_str(obj, istr(name)), .key = args[j].handle, .nth = relocs.count, }; @@ -2555,7 +2536,7 @@ static void cgen_aot_descr(llvm_obj_t *obj, cgen_func_t *func) const cgen_reloc_t r = { .kind = RELOC_FOREIGN, - .str = llvm_const_string(obj, tb_get(tb)), + .str = cgen_reloc_str(obj, tb_get(tb)), .key = (uintptr_t)args[j].foreign, .nth = relocs.count, }; @@ -2569,7 +2550,7 @@ static void cgen_aot_descr(llvm_obj_t *obj, cgen_func_t *func) const char *map[] = { "stmt", "branch", "toggle", "expr" }; const cgen_reloc_t r = { .kind = RELOC_COVER, - .str = llvm_const_string(obj, map[kind]), + .str = cgen_reloc_str(obj, map[kind]), .key = kind, .nth = relocs.count, }; @@ -2591,7 +2572,7 @@ static void cgen_aot_descr(llvm_obj_t *obj, cgen_func_t *func) for (int i = 0; i < relocs.count; i++) { LLVMValueRef fields[] = { llvm_int32(obj, relocs.items[i].kind), - relocs.items[i].str ?: LLVMConstNull(obj->types[LLVM_PTR]) + relocs.items[i].str ?: llvm_intptr(obj, 0) }; reloc_elems[i] = LLVMConstNamedStruct(obj->types[LLVM_AOT_RELOC], fields, ARRAY_LEN(fields)); @@ -2602,7 +2583,7 @@ static void cgen_aot_descr(llvm_obj_t *obj, cgen_func_t *func) LLVMTypeRef ftypes[] = { obj->types[LLVM_PTR], // Entry function - obj->types[LLVM_INT64], // FFI spec + obj->types[LLVM_PTR], // String table obj->types[LLVM_PTR], // JIT pack buffer obj->types[LLVM_PTR], // Constant pool func->reloc_type // Relocations list @@ -2618,7 +2599,7 @@ static void cgen_aot_descr(llvm_obj_t *obj, cgen_func_t *func) LLVMValueRef fields[] = { PTR(func->llvmfn), - llvm_int64(obj, 0), // XXX: really needed? + PTR(obj->strtab), PTR(irbuf), PTR(func->cpool), reloc_array, @@ -3334,12 +3315,12 @@ void jit_register_llvm_plugin(jit_t *j) llvm_obj_t *llvm_obj_new(const char *name) { llvm_obj_t *obj = xcalloc(sizeof(llvm_obj_t)); - obj->context = LLVMContextCreate(); - obj->module = LLVMModuleCreateWithNameInContext(name, obj->context); - obj->builder = LLVMCreateBuilderInContext(obj->context); - obj->target = llvm_target_machine(LLVMRelocPIC, LLVMCodeModelDefault); - obj->data_ref = LLVMCreateTargetDataLayout(obj->target); - obj->jitpack = jit_pack_new(); + obj->context = LLVMContextCreate(); + obj->module = LLVMModuleCreateWithNameInContext(name, obj->context); + obj->builder = LLVMCreateBuilderInContext(obj->context); + obj->target = llvm_target_machine(LLVMRelocPIC, LLVMCodeModelDefault); + obj->data_ref = LLVMCreateTargetDataLayout(obj->target); + obj->pack_writer = pack_writer_new(); #if ENABLE_DWARF obj->debuginfo = LLVMCreateDIBuilderDisallowUnresolved(obj->module); @@ -3362,6 +3343,11 @@ llvm_obj_t *llvm_obj_new(const char *name) #endif #endif + obj->strtab = LLVMAddGlobal(obj->module, obj->types[LLVM_STRTAB], + "placeholder_strtab"); + LLVMSetGlobalConstant(obj->strtab, true); + LLVMSetLinkage(obj->strtab, LLVMPrivateLinkage); + return obj; } @@ -3419,6 +3405,21 @@ void llvm_obj_finalise(llvm_obj_t *obj, llvm_opt_level_t olevel) LLVM_MUL_OVERFLOW_U8); } + const char *strtab; + size_t len; + pack_writer_string_table(obj->pack_writer, &strtab, &len); + + LLVMValueRef init = + LLVMConstStringInContext(obj->context, strtab, len - 1, false); + LLVMTypeRef array_type = LLVMArrayType(obj->types[LLVM_INT8], len); + LLVMValueRef global = LLVMAddGlobal(obj->module, array_type, "strtab"); + LLVMSetGlobalConstant(global, true); + LLVMSetInitializer(global, init); + LLVMSetLinkage(global, LLVMPrivateLinkage); + + LLVMReplaceAllUsesWith(obj->strtab, global); + LLVMDeleteGlobal(obj->strtab); + DWARF_ONLY(LLVMDIBuilderFinalize(obj->debuginfo)); llvm_dump_module(obj->module, "initial"); @@ -3440,8 +3441,7 @@ void llvm_obj_emit(llvm_obj_t *obj, const char *path) LLVMDisposeModule(obj->module); LLVMContextDispose(obj->context); - shash_free(obj->string_pool); - jit_pack_free(obj->jitpack); + pack_writer_free(obj->pack_writer); free(obj); } diff --git a/src/jit/jit-pack.c b/src/jit/jit-pack.c index f60ccbf7..3b0b40bb 100644 --- a/src/jit/jit-pack.c +++ b/src/jit/jit-pack.c @@ -30,262 +30,237 @@ #include #include -typedef enum { - PF_OWNED, - PF_BORROWED -} owner_kind_t; - typedef A(const char *) string_list_t; typedef struct { - ident_t name; - owner_kind_t kind; - shash_t *strtab; - string_list_t strings; - unsigned nextstr; - size_t cpoolsz; - uint8_t *cpool; - size_t bufsz; - uint8_t *wptr; - uint8_t *buf; + ident_t name; + const char *strtab; + uint8_t *buf; + uint8_t *rptr; + uint8_t *cpool; } pack_func_t; struct _jit_pack { chash_t *funcs; }; -jit_pack_t *jit_pack_new(void) -{ - jit_pack_t *jp = xcalloc(sizeof(struct _jit_pack)); - jp->funcs = chash_new(256); - - return jp; -} - -static void pack_func_free(const void *key, void *value) -{ - pack_func_t *pf = value; - - if (pf->kind == PF_OWNED) { - free(pf->buf); - free(pf->cpool); - } - - free(pf); -} +typedef struct _pack_writer { + text_buf_t *strtab; + size_t bufsz; + uint8_t *wptr; + uint8_t *buf; +} pack_writer_t; -void jit_pack_free(jit_pack_t *jp) -{ - chash_iter(jp->funcs, pack_func_free); - chash_free(jp->funcs); - free(jp); -} +//////////////////////////////////////////////////////////////////////////////// +// JIT bytecode serialisation -static inline void pack_grow(pack_func_t *pf, size_t minsz) +static inline void pack_grow(pack_writer_t *pw, size_t minsz) { - const size_t used = pf->wptr - pf->buf; - if (unlikely(used + minsz > pf->bufsz)) { - pf->bufsz = MAX(pf->bufsz + minsz, (3 * pf->bufsz) / 2); - pf->buf = xrealloc(pf->buf, pf->bufsz); - pf->wptr = pf->buf + used; + const size_t used = pw->wptr - pw->buf; + if (unlikely(used + minsz > pw->bufsz)) { + pw->bufsz = MAX(pw->bufsz + minsz, (3 * pw->bufsz) / 2); + pw->buf = xrealloc(pw->buf, pw->bufsz); + pw->wptr = pw->buf + used; } } -static inline void pack_u8(pack_func_t *pf, uint8_t value) +static inline void pack_u8(pack_writer_t *pw, uint8_t value) { - pack_grow(pf, 1); - *pf->wptr++ = value; + pack_grow(pw, 1); + *pw->wptr++ = value; } -static inline void pack_u16(pack_func_t *pf, uint16_t value) +static inline void pack_u16(pack_writer_t *pw, uint16_t value) { - pack_grow(pf, 2); - *pf->wptr++ = value & 0xff; - *pf->wptr++ = value >> 8; + pack_grow(pw, 2); + *pw->wptr++ = value & 0xff; + *pw->wptr++ = value >> 8; } -static void pack_uint(pack_func_t *pf, uint64_t value) +static void pack_uint(pack_writer_t *pw, uint64_t value) { - pack_grow(pf, 10); + pack_grow(pw, 10); do { uint8_t byte = value & 0x7f; value >>= 7; if (value) byte |= 0x80; - *pf->wptr++ = byte; + *pw->wptr++ = byte; } while (value); } -static inline void pack_int(pack_func_t *pf, int64_t value) +static inline void pack_int(pack_writer_t *pw, int64_t value) { uint64_t zz = (value << 1) ^ (value >> 63); // Zig-zag encoding - pack_uint(pf, zz); + pack_uint(pw, zz); } -static void pack_str(pack_func_t *pf, const char *str) +static void pack_str(pack_writer_t *pw, const char *str) { - if (str == NULL) - pack_int(pf, 0); - else { - const uintptr_t idx = (uintptr_t)shash_get(pf->strtab, str); - if (idx == 0) { - const int len = strlen(str); - pack_int(pf, -len); - pack_grow(pf, len + 1); - memcpy(pf->wptr, str, len + 1); - pf->wptr += len + 1; - - shash_put(pf->strtab, str, (void *)(uintptr_t)pf->nextstr++); - } - else - pack_int(pf, idx); - } + pack_uint(pw, pack_writer_get_string(pw, str)); } -static void pack_loc(pack_func_t *pf, const loc_t *loc) +static void pack_loc(pack_writer_t *pw, const loc_t *loc) { - pack_str(pf, loc_file_str(loc)); - pack_uint(pf, loc->first_line); - pack_uint(pf, loc->first_column); - pack_uint(pf, loc->line_delta); - pack_uint(pf, loc->column_delta); + pack_str(pw, loc_file_str(loc)); + pack_uint(pw, loc->first_line); + pack_uint(pw, loc->first_column); + pack_uint(pw, loc->line_delta); + pack_uint(pw, loc->column_delta); } -static inline void pack_reg(pack_func_t *pf, jit_reg_t reg) +static inline void pack_reg(pack_writer_t *pw, jit_reg_t reg) { - pack_uint(pf, reg == JIT_REG_INVALID ? 0 : reg + 1); + pack_uint(pw, reg == JIT_REG_INVALID ? 0 : reg + 1); } -static inline void pack_double(pack_func_t *pf, double value) +static inline void pack_double(pack_writer_t *pw, double value) { jit_scalar_t u = { .real = value }; - pack_uint(pf, u.integer); + pack_uint(pw, u.integer); } -static void pack_tree(pack_func_t *pf, tree_t tree) +static void pack_tree(pack_writer_t *pw, tree_t tree) { ident_t unit; ptrdiff_t offset; tree_locus(tree, &unit, &offset); - pack_str(pf, istr(unit)); - pack_uint(pf, offset); + pack_str(pw, istr(unit)); + pack_uint(pw, offset); } -static void pack_handle(pack_func_t *pf, jit_t *j, jit_handle_t handle) +static void pack_handle(pack_writer_t *pw, jit_t *j, jit_handle_t handle) { if (handle == JIT_HANDLE_INVALID) - pack_str(pf, NULL); + pack_str(pw, NULL); else - pack_str(pf, istr(jit_get_func(j, handle)->name)); + pack_str(pw, istr(jit_get_func(j, handle)->name)); } -static void pack_value(pack_func_t *pf, jit_t *j, jit_value_t value) +static void pack_value(pack_writer_t *pw, jit_t *j, jit_value_t value) { - pack_u8(pf, value.kind); + pack_u8(pw, value.kind); switch (value.kind) { case JIT_VALUE_INVALID: break; case JIT_VALUE_REG: - pack_reg(pf, value.reg); + pack_reg(pw, value.reg); break; case JIT_VALUE_INT64: - pack_int(pf, value.int64); + pack_int(pw, value.int64); break; case JIT_VALUE_DOUBLE: - pack_double(pf, value.dval); + pack_double(pw, value.dval); break; case JIT_ADDR_REG: - pack_reg(pf, value.reg); - pack_uint(pf, value.disp); + pack_reg(pw, value.reg); + pack_uint(pw, value.disp); break; case JIT_ADDR_CPOOL: - pack_uint(pf, value.disp); + pack_uint(pw, value.disp); break; case JIT_ADDR_ABS: case JIT_ADDR_COVER: - pack_uint(pf, value.int64); + pack_uint(pw, value.int64); break; case JIT_VALUE_LABEL: - pack_uint(pf, value.label); + pack_uint(pw, value.label); break; case JIT_VALUE_HANDLE: - pack_handle(pf, j, value.handle); + pack_handle(pw, j, value.handle); break; case JIT_VALUE_EXIT: - pack_u8(pf, value.exit); + pack_u8(pw, value.exit); break; case JIT_VALUE_LOC: - pack_loc(pf, &(value.loc)); + pack_loc(pw, &(value.loc)); break; case JIT_VALUE_TREE: - pack_tree(pf, value.tree); + pack_tree(pw, value.tree); break; case JIT_VALUE_FOREIGN: - pack_str(pf, istr(ffi_get_sym(value.foreign))); + pack_str(pw, istr(ffi_get_sym(value.foreign))); break; default: fatal_trace("cannot handle value kind %d in pack_value", value.kind); } } -static void pack_func(pack_func_t *pf, jit_t *j, jit_func_t *f) +static void pack_func(pack_writer_t *pw, jit_t *j, jit_func_t *f) { - pf->name = f->name; - pf->cpoolsz = f->cpoolsz; - - pf->cpool = xmalloc(f->cpoolsz); - memcpy(pf->cpool, f->cpool, f->cpoolsz); - - pack_uint(pf, f->nirs); - pack_uint(pf, f->nregs); - pack_uint(pf, f->nvars); - pack_uint(pf, f->cpoolsz); - pack_uint(pf, f->spec.bits); // XXX: need a function to pack jit_foreign_t - pack_uint(pf, f->framesz); + pack_uint(pw, f->nirs); + pack_uint(pw, f->nregs); + pack_uint(pw, f->nvars); + pack_uint(pw, f->cpoolsz); + pack_uint(pw, f->spec.bits); // XXX: need a function to pack jit_foreign_t + pack_uint(pw, f->framesz); for (int i = 0; i < f->nvars; i++) { - pack_str(pf, istr(f->linktab[i].name)); - pack_uint(pf, f->linktab[i].offset); + pack_str(pw, istr(f->linktab[i].name)); + pack_uint(pw, f->linktab[i].offset); } ident_t unit; ptrdiff_t offset; object_locus(f->object, &unit, &offset); - pack_str(pf, istr(unit)); - pack_uint(pf, offset); + pack_str(pw, istr(unit)); + pack_uint(pw, offset); for (int i = 0; i < f->nirs; i++) { jit_ir_t *ir = &(f->irbuf[i]); const uint16_t enc = (ir->op << 8) | (ir->size << 5) | (ir->target << 4) | ir->cc; - pack_u16(pf, enc); + pack_u16(pw, enc); - pack_reg(pf, ir->result); - pack_value(pf, j, ir->arg1); - pack_value(pf, j, ir->arg2); + pack_reg(pw, ir->result); + pack_value(pw, j, ir->arg1); + pack_value(pw, j, ir->arg2); } } -void jit_pack_encode(jit_pack_t *jp, jit_t *j, jit_handle_t handle) +pack_writer_t *pack_writer_new(void) +{ + pack_writer_t *pw = xcalloc(sizeof(pack_writer_t)); + pw->strtab = tb_new(); + pw->bufsz = 512; + pw->buf = xmalloc(pw->bufsz); + pw->wptr = pw->buf; + + tb_append(pw->strtab, '\0'); // Null string is index zero + + return pw; +} + +unsigned pack_writer_get_string(pack_writer_t *pw, const char *str) +{ + if (str == NULL) + return 0; + else { + const char *strtab = tb_get(pw->strtab); + const size_t tlen = tb_len(pw->strtab); + const size_t slen = strlen(str) + 1; + const char *exist = memmem(strtab, tlen, str, slen); + if (exist != NULL) + return exist - strtab; + else { + tb_catn(pw->strtab, str, slen); + return tlen; + } + } +} + +void pack_writer_emit(pack_writer_t *pw, jit_t *j, jit_handle_t handle, + uint8_t **buf, size_t *size) { jit_func_t *f = jit_get_func(j, handle); jit_fill_irbuf(f); - pack_func_t *pf = xcalloc(sizeof(pack_func_t)); - pf->kind = PF_OWNED; - pf->bufsz = f->nirs * 10; - pf->buf = xmalloc(pf->bufsz); - pf->wptr = pf->buf; - pf->strtab = shash_new(128); - pf->nextstr = 1; - - pack_func(pf, j, f); + assert(pw->wptr == pw->buf); - shash_free(pf->strtab); - pf->strtab = NULL; + pack_func(pw, j, f); z_stream strm; strm.zalloc = Z_NULL; @@ -295,7 +270,7 @@ void jit_pack_encode(jit_pack_t *jp, jit_t *j, jit_handle_t handle) if (deflateInit(&strm, Z_DEFAULT_COMPRESSION) != Z_OK) fatal_trace("deflateInit failed"); - const size_t ubufsz = pf->wptr - pf->buf; + const size_t ubufsz = pw->wptr - pw->buf; const size_t zbufsz = deflateBound(&strm, ubufsz) + 4; uint8_t *zbuf = xmalloc(zbufsz); @@ -304,8 +279,8 @@ void jit_pack_encode(jit_pack_t *jp, jit_t *j, jit_handle_t handle) zbuf[2] = (ubufsz >> 16) & 0xff; zbuf[3] = (ubufsz >> 24) & 0xff; - strm.next_in = pf->buf; - strm.avail_in = pf->wptr - pf->buf; + strm.next_in = pw->buf; + strm.avail_in = pw->wptr - pw->buf; strm.avail_out = zbufsz - 4; strm.next_out = zbuf + 4; @@ -314,45 +289,58 @@ void jit_pack_encode(jit_pack_t *jp, jit_t *j, jit_handle_t handle) deflateEnd(&strm); - free(pf->buf); - pf->buf = zbuf; - pf->bufsz = zbufsz - strm.avail_out; + pw->wptr = pw->buf; - chash_put(jp->funcs, f->name, pf); + *buf = zbuf; + *size = zbufsz - strm.avail_out; } -void jit_pack_vcode(jit_pack_t *jp, jit_t *j, vcode_unit_t vu) +void pack_writer_string_table(pack_writer_t *pw, const char **tab, size_t *size) { - vcode_select_unit(vu); - jit_handle_t h = jit_lazy_compile(j, vcode_unit_name()); - jit_pack_encode(jp, j, h); + *tab = tb_get(pw->strtab); + *size = tb_len(pw->strtab); +} - for (vcode_unit_t it = vcode_unit_child(vu); - it != NULL; - it = vcode_unit_next(it)) { - jit_pack_vcode(jp, j, it); - } +void pack_writer_free(pack_writer_t *pw) +{ + free(pw->buf); + tb_free(pw->strtab); + free(pw); } -const uint8_t *jit_pack_get(jit_pack_t *jp, ident_t name, size_t *size) +//////////////////////////////////////////////////////////////////////////////// +// JIT bytecode loader + +jit_pack_t *jit_pack_new(void) { - pack_func_t *pf = chash_get(jp->funcs, name); - if (pf == NULL) - return NULL; + jit_pack_t *jp = xcalloc(sizeof(struct _jit_pack)); + jp->funcs = chash_new(256); - *size = pf->bufsz; - return pf->buf; + return jp; +} + +static void pack_func_free(const void *key, void *value) +{ + pack_func_t *pf = value; + free(pf); +} + +void jit_pack_free(jit_pack_t *jp) +{ + chash_iter(jp->funcs, pack_func_free); + chash_free(jp->funcs); + free(jp); } void jit_pack_put(jit_pack_t *jp, ident_t name, const uint8_t *cpool, - const uint8_t *buf) + const char *strtab, const uint8_t *buf) { assert(chash_get(jp->funcs, name) == NULL); pack_func_t *pf = xcalloc(sizeof(pack_func_t)); - pf->kind = PF_BORROWED; - pf->buf = (uint8_t *)buf; - pf->cpool = (uint8_t *)cpool; + pf->buf = (uint8_t *)buf; + pf->cpool = (uint8_t *)cpool; + pf->strtab = strtab; chash_put(jp->funcs, name, pf); } @@ -364,7 +352,7 @@ static uint64_t unpack_uint(pack_func_t *pf) uint8_t byte; do { - byte = *pf->wptr++; + byte = *pf->rptr++; dec[nbytes++] = byte & 0x7f; } while (byte & 0x80); @@ -386,8 +374,8 @@ static int64_t unpack_int(pack_func_t *pf) static inline uint16_t unpack_u16(pack_func_t *pf) { uint16_t value = 0; - value |= *pf->wptr++; - value |= *pf->wptr++ << 8; + value |= *pf->rptr++; + value |= *pf->rptr++ << 8; return value; } @@ -406,17 +394,8 @@ static inline double unpack_double(pack_func_t *pf) static const char *unpack_str(pack_func_t *pf) { - const int64_t enc = unpack_int(pf); - if (enc == 0) - return NULL; - else if (enc < 0) { - const char *str = (char *)pf->wptr; - pf->wptr += -enc + 1; - APUSH(pf->strings, str); - return str; - } - else - return AGET(pf->strings, enc - 1); + const unsigned off = unpack_uint(pf); + return pf->strtab + off; } static loc_t unpack_loc(pack_func_t *pf) @@ -450,7 +429,7 @@ static jit_handle_t unpack_handle(pack_func_t *pf, jit_t *j) static jit_value_t unpack_value(pack_func_t *pf, jit_t *j) { jit_value_t value; - value.kind = *pf->wptr++; + value.kind = *pf->rptr++; switch (value.kind) { case JIT_VALUE_INVALID: @@ -482,7 +461,7 @@ static jit_value_t unpack_value(pack_func_t *pf, jit_t *j) value.handle = unpack_handle(pf, j); break; case JIT_VALUE_EXIT: - value.exit = *pf->wptr++; + value.exit = *pf->rptr++; break; case JIT_VALUE_LOC: value.loc = unpack_loc(pf); @@ -529,7 +508,7 @@ bool jit_pack_fill(jit_pack_t *jp, jit_t *j, jit_func_t *f) if (inflate(&strm, Z_FINISH) != Z_STREAM_END) fatal_trace("inflate failed"); - pf->wptr = ubuf; + pf->rptr = ubuf; f->nirs = unpack_uint(pf); f->nregs = unpack_uint(pf); @@ -572,8 +551,7 @@ bool jit_pack_fill(jit_pack_t *jp, jit_t *j, jit_func_t *f) f->cpool = pf->cpool; - ACLEAR(pf->strings); - pf->wptr = NULL; + pf->rptr = NULL; store_release(&(f->state), JIT_FUNC_READY); return true; diff --git a/src/jit/jit-priv.h b/src/jit/jit-priv.h index 1e85ad77..79b42e4e 100644 --- a/src/jit/jit-priv.h +++ b/src/jit/jit-priv.h @@ -327,6 +327,8 @@ typedef struct { bool overflow; } code_blob_t; +typedef struct _pack_writer pack_writer_t; + #define JIT_MAX_ARGS 64 typedef struct _jit_interp jit_interp_t; @@ -379,9 +381,16 @@ void code_blob_patch(code_blob_t *blob, jit_label_t label, code_patch_fn_t fn); void code_load_object(code_blob_t *blob, const void *data, size_t size); bool jit_pack_fill(jit_pack_t *jp, jit_t *j, jit_func_t *f); -const uint8_t *jit_pack_get(jit_pack_t *jp, ident_t name, size_t *size); void jit_pack_put(jit_pack_t *jp, ident_t name, const uint8_t *cpool, - const uint8_t *buf); + const char *strtab, const uint8_t *buf); + +pack_writer_t *pack_writer_new(void); +void pack_writer_emit(pack_writer_t *pw, jit_t *j, jit_handle_t handle, + uint8_t **buf, size_t *size); +unsigned pack_writer_get_string(pack_writer_t *pw, const char *str); +void pack_writer_string_table(pack_writer_t *pw, const char **tab, + size_t *size); +void pack_writer_free(pack_writer_t *pw); void __nvc_do_exit(jit_exit_t which, jit_anchor_t *anchor, jit_scalar_t *args, tlab_t *tlab); diff --git a/src/rt/rt.h b/src/rt/rt.h index 3b7c63f7..e2ea11a2 100644 --- a/src/rt/rt.h +++ b/src/rt/rt.h @@ -23,7 +23,7 @@ #include -#define RT_ABI_VERSION 12 +#define RT_ABI_VERSION 13 #define RT_ALIGN_MASK 0x7 #define RT_MULTITHREADED 0 -- 2.39.2