From 1745c08118eab920908799ee09bc71edd46640e7 Mon Sep 17 00:00:00 2001 From: Nick Gasson Date: Sun, 21 Apr 2024 11:21:16 +0100 Subject: [PATCH] Use saturating addition for coverage counters. Issue #880 --- src/jit/jit-dump.c | 2 +- src/jit/jit-interp.c | 20 +++++++++++++++++- src/jit/jit-irgen.c | 12 +++++++---- src/jit/jit-llvm.c | 49 ++++++++++++++++++++++++++++++++++++++++++++ src/jit/jit-priv.h | 1 + 5 files changed, 78 insertions(+), 6 deletions(-) diff --git a/src/jit/jit-dump.c b/src/jit/jit-dump.c index 9b6674d6..c52ea9a8 100644 --- a/src/jit/jit-dump.c +++ b/src/jit/jit-dump.c @@ -45,7 +45,7 @@ const char *jit_op_name(jit_op_t op) static const char *names[] = { "$COPY", "$GALLOC", "$EXIT", "$FEXP", "$EXP", "$BZERO", "$GETPRIV", "$PUTPRIV", "$LALLOC", "$SALLOC", "$CASE", - "$TRIM", "$MOVE", "$MEMSET", "$REEXEC", + "$TRIM", "$MOVE", "$MEMSET", "$REEXEC", "$SADD", }; assert(op - __MACRO_BASE < ARRAY_LEN(names)); return names[op - __MACRO_BASE]; diff --git a/src/jit/jit-interp.c b/src/jit/jit-interp.c index c01dde94..71871753 100644 --- a/src/jit/jit-interp.c +++ b/src/jit/jit-interp.c @@ -1,5 +1,5 @@ // -// Copyright (C) 2022 Nick Gasson +// Copyright (C) 2022-2024 Nick Gasson // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by @@ -869,6 +869,21 @@ static void interp_reexec(jit_interp_t *state, jit_ir_t *ir) (*entry)(state->func, state->anchor->caller, state->args, state->tlab); } +static void interp_sadd(jit_interp_t *state, jit_ir_t *ir) +{ + const void *ptr = interp_get_pointer(state, ir->arg1); + const int64_t addend = interp_get_int(state, ir->arg2); + +#define SADD(type) do { \ + u##type cur = *(u##type *)ptr, next = cur + addend; \ + if (next < cur) \ + next = (u##type)INT64_C(-1); \ + *(u##type *)ptr = next; \ + } while (0) + + FOR_EACH_SIZE(ir->size, SADD); +} + static void interp_loop(jit_interp_t *state) { for (;;) { @@ -1033,6 +1048,9 @@ static void interp_loop(jit_interp_t *state) case MACRO_REEXEC: interp_reexec(state, ir); return; + case MACRO_SADD: + interp_sadd(state, ir); + break; default: interp_dump(state); fatal_trace("cannot interpret opcode %s", jit_op_name(ir->op)); diff --git a/src/jit/jit-irgen.c b/src/jit/jit-irgen.c index a09190eb..1f8f378f 100644 --- a/src/jit/jit-irgen.c +++ b/src/jit/jit-irgen.c @@ -694,6 +694,13 @@ static void macro_reexec(jit_irgen_t *g) g->flags = VCODE_INVALID_REG; } +static void macro_sadd(jit_irgen_t *g, jit_size_t sz, jit_value_t addr, + jit_value_t addend) +{ + irgen_emit_binary(g, MACRO_SADD, sz, JIT_CC_NONE, JIT_REG_INVALID, + addr, addend); +} + //////////////////////////////////////////////////////////////////////////////// // Vcode to JIT IR lowering @@ -3309,10 +3316,7 @@ static void irgen_op_cover_increment(jit_irgen_t *g, int op) uint32_t tag = vcode_get_tag(op); jit_value_t mem = jit_addr_from_cover_tag(tag); - // XXX: this should be atomic - jit_value_t cur = j_load(g, JIT_SZ_32, mem); - jit_value_t inc = j_add(g, cur, jit_value_from_int64(1)); - j_store(g, JIT_SZ_32, inc, mem); + macro_sadd(g, JIT_SZ_32, mem, jit_value_from_int64(1)); } static void irgen_op_cover_expr(jit_irgen_t *g, int op) diff --git a/src/jit/jit-llvm.c b/src/jit/jit-llvm.c index c10b8c0a..331de1f7 100644 --- a/src/jit/jit-llvm.c +++ b/src/jit/jit-llvm.c @@ -113,6 +113,11 @@ typedef enum { LLVM_EXP_OVERFLOW_U32, LLVM_EXP_OVERFLOW_U64, + LLVM_ADD_SAT_U8, + LLVM_ADD_SAT_U16, + LLVM_ADD_SAT_U32, + LLVM_ADD_SAT_U64, + LLVM_MEMSET_U8, LLVM_MEMSET_U16, LLVM_MEMSET_U32, @@ -709,6 +714,27 @@ static LLVMValueRef llvm_get_fn(llvm_obj_t *obj, llvm_fn_t which) } break; + case LLVM_ADD_SAT_U8: + case LLVM_ADD_SAT_U16: + case LLVM_ADD_SAT_U32: + case LLVM_ADD_SAT_U64: + { + jit_size_t sz = which - LLVM_ADD_SAT_U8; + LLVMTypeRef int_type = obj->types[LLVM_INT8 + sz]; + LLVMTypeRef args[] = { int_type, int_type }; + obj->fntypes[which] = LLVMFunctionType(int_type, args, + ARRAY_LEN(args), false); + + static const char *names[] = { + "llvm.uadd.sat.i8", + "llvm.uadd.sat.i16", + "llvm.uadd.sat.i32", + "llvm.uadd.sat.i64" + }; + fn = llvm_add_fn(obj, names[sz], obj->fntypes[which]); + } + break; + case LLVM_MEMSET_U8: case LLVM_MEMSET_U16: case LLVM_MEMSET_U32: @@ -2277,6 +2303,25 @@ static void cgen_macro_reexec(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) LLVMBuildRetVoid(obj->builder); } +static void cgen_macro_sadd(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) +{ + LLVMValueRef ptr = cgen_coerce_value(obj, cgb, ir->arg1, LLVM_PTR); + llvm_type_t type = LLVM_INT8 + ir->size; + +#ifndef LLVM_HAS_OPAQUE_POINTERS + LLVMTypeRef ptr_type = LLVMPointerType(obj->types[type], 0); + ptr = LLVMBuildPointerCast(obj->builder, ptr, ptr_type, ""); +#endif + + LLVMValueRef cur = LLVMBuildLoad2(obj->builder, obj->types[type], ptr, ""); + LLVMValueRef addend = cgen_coerce_value(obj, cgb, ir->arg2, type); + + LLVMValueRef args[] = { cur, addend }; + LLVMValueRef sat = llvm_call_fn(obj, LLVM_ADD_SAT_U8 + ir->size, args, 2); + + LLVMBuildStore(obj->builder, sat, ptr); +} + static void cgen_ir(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) { switch (ir->op) { @@ -2437,6 +2482,9 @@ static void cgen_ir(llvm_obj_t *obj, cgen_block_t *cgb, jit_ir_t *ir) case MACRO_REEXEC: cgen_macro_reexec(obj, cgb, ir); break; + case MACRO_SADD: + cgen_macro_sadd(obj, cgb, ir); + break; default: cgen_abort(cgb, ir, "cannot generate LLVM for %s", jit_op_name(ir->op)); } @@ -2704,6 +2752,7 @@ static void cgen_pointer_mask(cgen_func_t *func) case J_LOAD: case J_ULOAD: case MACRO_BZERO: + case MACRO_SADD: cgen_must_be_pointer(func, ir->arg1); break; case J_STORE: diff --git a/src/jit/jit-priv.h b/src/jit/jit-priv.h index bafee182..c0037bb9 100644 --- a/src/jit/jit-priv.h +++ b/src/jit/jit-priv.h @@ -85,6 +85,7 @@ typedef enum { MACRO_MOVE, MACRO_MEMSET, MACRO_REEXEC, + MACRO_SADD, } jit_op_t; typedef enum { -- 2.39.2