diff options
Diffstat (limited to 'tools/testing/selftests/acct')
| -rw-r--r-- | tools/testing/selftests/acct/.gitignore | 4 | ||||
| -rw-r--r-- | tools/testing/selftests/acct/Makefile | 16 | ||||
| -rw-r--r-- | tools/testing/selftests/acct/cgroupstats.c | 231 | ||||
| -rw-r--r-- | tools/testing/selftests/acct/netlink_helper.c | 116 | ||||
| -rw-r--r-- | tools/testing/selftests/acct/netlink_helper.h | 44 | ||||
| -rw-r--r-- | tools/testing/selftests/acct/taskstats_fill_stats_tgid.c | 257 |
6 files changed, 666 insertions, 2 deletions
diff --git a/tools/testing/selftests/acct/.gitignore b/tools/testing/selftests/acct/.gitignore index 7e78aac19038..fe0896f54e15 100644 --- a/tools/testing/selftests/acct/.gitignore +++ b/tools/testing/selftests/acct/.gitignore @@ -1,3 +1,5 @@ acct_syscall +taskstats_fill_stats_tgid +cgroupstats config -process_log
\ No newline at end of file +process_log diff --git a/tools/testing/selftests/acct/Makefile b/tools/testing/selftests/acct/Makefile index 7e025099cf65..93a11a28a636 100644 --- a/tools/testing/selftests/acct/Makefile +++ b/tools/testing/selftests/acct/Makefile @@ -1,5 +1,19 @@ # SPDX-License-Identifier: GPL-2.0 TEST_GEN_PROGS := acct_syscall +TEST_GEN_PROGS += taskstats_fill_stats_tgid +TEST_GEN_PROGS += cgroupstats + +NETLINK_HELPER_PROGS := cgroupstats taskstats_fill_stats_tgid + CFLAGS += -Wall +LDLIBS += -lpthread + +include ../lib.mk + +$(NETLINK_HELPER_PROGS): %: %.c netlink_helper.c netlink_helper.h + $(call msg,CC,,$@) + $(Q)$(LINK.c) $< netlink_helper.c $(LDLIBS) -o $@ -include ../lib.mk
\ No newline at end of file +$(addprefix $(OUTPUT)/,$(NETLINK_HELPER_PROGS)): $(OUTPUT)/%: %.c netlink_helper.c netlink_helper.h + $(call msg,CC,,$@) + $(Q)$(LINK.c) $< netlink_helper.c $(LDLIBS) -o $@ diff --git a/tools/testing/selftests/acct/cgroupstats.c b/tools/testing/selftests/acct/cgroupstats.c new file mode 100644 index 000000000000..0b421a4ca72b --- /dev/null +++ b/tools/testing/selftests/acct/cgroupstats.c @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE + +#include <errno.h> +#include <fcntl.h> +#include <linux/cgroupstats.h> +#include <linux/genetlink.h> +#include <linux/netlink.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mount.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <unistd.h> + +#include "netlink_helper.h" +#include "kselftest.h" + +static int send_cgroupstats_cmd(int fd, int family_id, uint32_t cgroup_fd, + int flags) +{ + struct { + struct nlmsghdr nlh; + struct genlmsghdr genl; + char buf[256]; + } req = { 0 }; + struct nlattr *na; + + req.nlh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN); + req.nlh.nlmsg_type = family_id; + req.nlh.nlmsg_flags = NLM_F_REQUEST | flags; + req.nlh.nlmsg_seq = 2; + req.nlh.nlmsg_pid = getpid(); + + req.genl.cmd = CGROUPSTATS_CMD_GET; + req.genl.version = 1; + + na = (struct nlattr *)((char *)&req + NLMSG_ALIGN(req.nlh.nlmsg_len)); + na->nla_type = CGROUPSTATS_CMD_ATTR_FD; + na->nla_len = NLA_HDRLEN + sizeof(cgroup_fd); + memcpy(nla_data(na), &cgroup_fd, sizeof(cgroup_fd)); + req.nlh.nlmsg_len = NLMSG_ALIGN(req.nlh.nlmsg_len) + NLA_ALIGN(na->nla_len); + + return send_request(fd, &req, req.nlh.nlmsg_len); +} + +/* + * Receive and decode a cgroupstats response. + * + * Returns: + * 0 — success, stats filled from CGROUPSTATS_CMD_NEW reply + * <0 — NLMSG_ERROR errno (e.g. -EBADF, -EINVAL) + */ +static int recv_cgroupstats_response(int fd, struct cgroupstats *stats) +{ + char resp[8192]; + struct nlmsghdr *nlh; + struct genlmsghdr *genl; + struct nlattr *na; + int len; + int rem; + + memset(stats, 0, sizeof(*stats)); + + len = recv(fd, resp, sizeof(resp), 0); + if (len < 0) + return -errno; + + for (nlh = (struct nlmsghdr *)resp; NLMSG_OK(nlh, len); + nlh = NLMSG_NEXT(nlh, len)) { + if (nlh->nlmsg_type == NLMSG_ERROR) { + struct nlmsgerr *err = NLMSG_DATA(nlh); + + return err->error; + } + + genl = (struct genlmsghdr *)NLMSG_DATA(nlh); + if (genl->cmd != CGROUPSTATS_CMD_NEW) + continue; + + rem = nlh->nlmsg_len - NLMSG_HDRLEN - GENL_HDRLEN; + na = (struct nlattr *)((char *)genl + GENL_HDRLEN); + while (nla_ok(na, rem)) { + if (na->nla_type == CGROUPSTATS_TYPE_CGROUP_STATS) { + memcpy(stats, nla_data(na), sizeof(*stats)); + return 0; + } + na = nla_next(na, &rem); + } + } + + return -EIO; +} + +/* mkdtemp() modifies the template in place, so this cannot be const. */ +static char cg_mountpoint[32]; +static bool cg_mounted; + +static int setup_cgroup_v1(void) +{ + strcpy(cg_mountpoint, "/tmp/cgstats_test_XXXXXX"); + + if (!mkdtemp(cg_mountpoint)) + return -errno; + + if (mount("cgstats_test", cg_mountpoint, "cgroup", 0, + "none,name=cgstats_test") < 0) { + int ret = -errno; + + rmdir(cg_mountpoint); + return ret; + } + + cg_mounted = true; + return 0; +} + +static void cleanup_cgroup_v1(void) +{ + if (!cg_mounted) + return; + umount2(cg_mountpoint, MNT_DETACH); + rmdir(cg_mountpoint); + cg_mounted = false; +} + +int main(void) +{ + struct cgroupstats stats; + uint64_t total_tasks; + int family_id; + int nl_fd; + int cg_fd; + int ret; + + ksft_print_header(); + + nl_fd = netlink_open(); + if (nl_fd < 0) + ksft_exit_skip("failed to open generic netlink socket: %s\n", + strerror(-nl_fd)); + + family_id = get_family_id(nl_fd, TASKSTATS_GENL_NAME); + if (family_id < 0) + ksft_exit_skip("taskstats generic netlink family unavailable: %s\n", + strerror(-family_id)); + + ksft_set_plan(3); + + /* + * Test 1: mount a private cgroup v1 hierarchy, query it, and + * verify the response contains sane task counts. If the test + * environment cannot create a private cgroup v1 mount, skip this + * case and continue with the unprivileged regression checks below. + */ + ret = setup_cgroup_v1(); + if (ret) { + ksft_test_result_skip("cgroupstats query: cannot mount cgroup v1: %s\n", + strerror(-ret)); + } else { + cg_fd = open(cg_mountpoint, O_RDONLY | O_DIRECTORY); + if (cg_fd < 0) { + ksft_test_result_fail("cgroupstats query: open mountpoint: %s\n", + strerror(errno)); + } else { + ret = send_cgroupstats_cmd(nl_fd, family_id, + (uint32_t)cg_fd, 0); + if (ret) { + ksft_test_result_fail("cgroupstats query: send: %s\n", + strerror(-ret)); + } else { + ret = recv_cgroupstats_response(nl_fd, &stats); + if (ret < 0) { + ksft_test_result_fail("cgroupstats query: %s\n", + strerror(-ret)); + } else { + total_tasks = (uint64_t)stats.nr_sleeping + + (uint64_t)stats.nr_running + + (uint64_t)stats.nr_stopped + + (uint64_t)stats.nr_uninterruptible + + (uint64_t)stats.nr_io_wait; + + ksft_print_msg("cgroupstats query: total_tasks=%llu\n", + (unsigned long long)total_tasks); + + ksft_test_result(total_tasks > 0, + "cgroupstats query returns valid stats\n"); + } + } + close(cg_fd); + } + } + cleanup_cgroup_v1(); + + /* + * Test 2: invalid fd without NLM_F_ACK. The kernel should + * return -EBADF via NLMSG_ERROR regardless of whether the + * client requested an explicit ACK. + */ + ret = send_cgroupstats_cmd(nl_fd, family_id, 0xFFFFFFFF, 0); + if (ret) + ksft_exit_fail_msg("send test 2 failed: %s\n", strerror(-ret)); + + ret = recv_cgroupstats_response(nl_fd, &stats); + ksft_print_msg("bad fd (no ACK): response=%d (%s)\n", + ret, ret < 0 ? strerror(-ret) : "unexpected success"); + ksft_test_result(ret == -EBADF, + "cgroupstats rejects bad fd without NLM_F_ACK\n"); + + /* + * Test 3: invalid fd with NLM_F_ACK. Same expectation as + * test 2, but exercised through a different netlink flag + * path in the kernel's ack/error handling. + */ + ret = send_cgroupstats_cmd(nl_fd, family_id, 0xFFFFFFFF, NLM_F_ACK); + if (ret) + ksft_exit_fail_msg("send test 3 failed: %s\n", strerror(-ret)); + + ret = recv_cgroupstats_response(nl_fd, &stats); + ksft_print_msg("bad fd (with ACK): response=%d (%s)\n", + ret, ret < 0 ? strerror(-ret) : "unexpected success"); + ksft_test_result(ret == -EBADF, + "cgroupstats rejects bad fd with NLM_F_ACK\n"); + + close(nl_fd); + ksft_finished(); + return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS; +} diff --git a/tools/testing/selftests/acct/netlink_helper.c b/tools/testing/selftests/acct/netlink_helper.c new file mode 100644 index 000000000000..3ed834f0e770 --- /dev/null +++ b/tools/testing/selftests/acct/netlink_helper.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <errno.h> +#include <stdint.h> +#include <string.h> +#include <sys/socket.h> +#include <sys/time.h> +#include <unistd.h> +#include <linux/genetlink.h> + +#include "netlink_helper.h" + +int netlink_open(void) +{ + struct timeval tv = { .tv_sec = ACCT_RCV_TIMEOUT_SEC }; + struct sockaddr_nl addr = { + .nl_family = AF_NETLINK, + .nl_pid = getpid(), + }; + int fd; + + fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC); + if (fd < 0) + return -errno; + + if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) < 0) { + int err = -errno; + + close(fd); + return err; + } + + if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + int err = -errno; + + close(fd); + return err; + } + + return fd; +} + +int send_request(int fd, void *buf, size_t len) +{ + struct sockaddr_nl addr = { + .nl_family = AF_NETLINK, + }; + + if (sendto(fd, buf, len, 0, (struct sockaddr *)&addr, sizeof(addr)) < 0) + return -errno; + + return 0; +} + +/* + * Resolve the generic netlink family ID for @name. + * Returns the family ID (>= 0) on success, negative errno on failure. + */ +int get_family_id(int fd, const char *name) +{ + struct { + struct nlmsghdr nlh; + struct genlmsghdr genl; + char buf[256]; + } req = { 0 }; + char resp[8192]; + struct nlmsghdr *nlh; + struct genlmsghdr *genl; + struct nlattr *na; + int len; + int rem; + int ret; + + req.nlh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN); + req.nlh.nlmsg_type = GENL_ID_CTRL; + req.nlh.nlmsg_flags = NLM_F_REQUEST; + req.nlh.nlmsg_seq = 1; + req.nlh.nlmsg_pid = getpid(); + + req.genl.cmd = CTRL_CMD_GETFAMILY; + req.genl.version = 1; + + na = (struct nlattr *)((char *)&req + NLMSG_ALIGN(req.nlh.nlmsg_len)); + na->nla_type = CTRL_ATTR_FAMILY_NAME; + na->nla_len = NLA_HDRLEN + strlen(name) + 1; + memcpy(nla_data(na), name, strlen(name) + 1); + req.nlh.nlmsg_len = NLMSG_ALIGN(req.nlh.nlmsg_len) + NLA_ALIGN(na->nla_len); + + ret = send_request(fd, &req, req.nlh.nlmsg_len); + if (ret) + return ret; + + len = recv(fd, resp, sizeof(resp), 0); + if (len < 0) + return -errno; + + for (nlh = (struct nlmsghdr *)resp; NLMSG_OK(nlh, len); + nlh = NLMSG_NEXT(nlh, len)) { + if (nlh->nlmsg_type == NLMSG_ERROR) { + struct nlmsgerr *err = NLMSG_DATA(nlh); + + return err->error ? err->error : -ENOENT; + } + + genl = (struct genlmsghdr *)NLMSG_DATA(nlh); + rem = nlh->nlmsg_len - NLMSG_HDRLEN - GENL_HDRLEN; + na = (struct nlattr *)((char *)genl + GENL_HDRLEN); + while (nla_ok(na, rem)) { + if (na->nla_type == CTRL_ATTR_FAMILY_ID) + return *(uint16_t *)nla_data(na); + na = nla_next(na, &rem); + } + } + + return -ENOENT; +} diff --git a/tools/testing/selftests/acct/netlink_helper.h b/tools/testing/selftests/acct/netlink_helper.h new file mode 100644 index 000000000000..0320729c4c06 --- /dev/null +++ b/tools/testing/selftests/acct/netlink_helper.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Shared generic netlink helpers for the acct selftests. + */ +#ifndef ACSELFTESTS_ACCT_NETLINK_HELPER_H +#define ACSELFTESTS_ACCT_NETLINK_HELPER_H + +#include <stdbool.h> +#include <linux/netlink.h> + +#ifndef NLA_ALIGNTO +#define NLA_ALIGNTO 4 +#define NLA_ALIGN(len) (((len) + NLA_ALIGNTO - 1) & ~(NLA_ALIGNTO - 1)) +#define NLA_HDRLEN ((int)NLA_ALIGN(sizeof(struct nlattr))) +#endif + +/* Fail an individual test case instead of hanging the whole binary. */ +#define ACCT_RCV_TIMEOUT_SEC 2 + +static inline void *nla_data(const struct nlattr *na) +{ + return (void *)((char *)na + NLA_HDRLEN); +} + +static inline bool nla_ok(const struct nlattr *na, int remaining) +{ + return remaining >= (int)sizeof(*na) && + na->nla_len >= sizeof(*na) && + na->nla_len <= remaining; +} + +static inline struct nlattr *nla_next(const struct nlattr *na, int *remaining) +{ + int aligned_len = NLA_ALIGN(na->nla_len); + + *remaining -= aligned_len; + return (struct nlattr *)((char *)na + aligned_len); +} + +int netlink_open(void); +int send_request(int fd, void *buf, size_t len); +int get_family_id(int fd, const char *name); + +#endif /* ACSELFTESTS_ACCT_NETLINK_HELPER_H */ diff --git a/tools/testing/selftests/acct/taskstats_fill_stats_tgid.c b/tools/testing/selftests/acct/taskstats_fill_stats_tgid.c new file mode 100644 index 000000000000..9a4c1554dee3 --- /dev/null +++ b/tools/testing/selftests/acct/taskstats_fill_stats_tgid.c @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE + +#include <errno.h> +#include <linux/genetlink.h> +#include <linux/netlink.h> +#include <linux/taskstats.h> +#include <pthread.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <time.h> +#include <unistd.h> + +#include "netlink_helper.h" +#include "kselftest.h" + +#define BUSY_NS (200ULL * 1000 * 1000) + +struct worker_ctx { + pthread_mutex_t lock; + pthread_cond_t cond; + bool ready; + bool release; +}; + +static unsigned long busy_sink; + +static uint64_t timespec_diff_ns(const struct timespec *start, + const struct timespec *end) +{ + return (uint64_t)(end->tv_sec - start->tv_sec) * 1000000000ULL + + (uint64_t)(end->tv_nsec - start->tv_nsec); +} + +static void burn_cpu_for_ns(uint64_t runtime_ns) +{ + struct timespec start, now; + unsigned long acc = 0; + + if (clock_gettime(CLOCK_MONOTONIC, &start)) { + perror("clock_gettime"); + exit(EXIT_FAILURE); + } + + do { + for (int i = 0; i < 100000; i++) + acc += i; + if (clock_gettime(CLOCK_MONOTONIC, &now)) { + perror("clock_gettime"); + exit(EXIT_FAILURE); + } + } while (timespec_diff_ns(&start, &now) < runtime_ns); + + busy_sink = acc; +} + +static int get_taskstats(int fd, int family_id, uint16_t attr_type, uint32_t id, + struct taskstats *stats) +{ + struct { + struct nlmsghdr nlh; + struct genlmsghdr genl; + char buf[256]; + } req = { 0 }; + char resp[16384]; + struct nlmsghdr *nlh; + struct genlmsghdr *genl; + struct nlattr *na; + struct nlattr *nested; + int len; + int rem; + int nrem; + int ret; + + memset(stats, 0, sizeof(*stats)); + + req.nlh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN); + req.nlh.nlmsg_type = family_id; + req.nlh.nlmsg_flags = NLM_F_REQUEST; + req.nlh.nlmsg_seq = 2; + req.nlh.nlmsg_pid = getpid(); + + req.genl.cmd = TASKSTATS_CMD_GET; + req.genl.version = 1; + + na = (struct nlattr *)((char *)&req + NLMSG_ALIGN(req.nlh.nlmsg_len)); + na->nla_type = attr_type; + na->nla_len = NLA_HDRLEN + sizeof(id); + memcpy(nla_data(na), &id, sizeof(id)); + req.nlh.nlmsg_len = NLMSG_ALIGN(req.nlh.nlmsg_len) + NLA_ALIGN(na->nla_len); + + ret = send_request(fd, &req, req.nlh.nlmsg_len); + if (ret) + return ret; + + len = recv(fd, resp, sizeof(resp), 0); + if (len < 0) + return -errno; + + for (nlh = (struct nlmsghdr *)resp; NLMSG_OK(nlh, len); + nlh = NLMSG_NEXT(nlh, len)) { + if (nlh->nlmsg_type == NLMSG_ERROR) { + struct nlmsgerr *err = NLMSG_DATA(nlh); + + return err->error ? err->error : -ENOENT; + } + + genl = (struct genlmsghdr *)NLMSG_DATA(nlh); + rem = nlh->nlmsg_len - NLMSG_HDRLEN - GENL_HDRLEN; + na = (struct nlattr *)((char *)genl + GENL_HDRLEN); + while (nla_ok(na, rem)) { + if (na->nla_type == TASKSTATS_TYPE_AGGR_PID || + na->nla_type == TASKSTATS_TYPE_AGGR_TGID) { + nested = (struct nlattr *)nla_data(na); + nrem = na->nla_len - NLA_HDRLEN; + while (nla_ok(nested, nrem)) { + if (nested->nla_type == TASKSTATS_TYPE_STATS) { + memcpy(stats, nla_data(nested), + sizeof(*stats)); + return 0; + } + nested = nla_next(nested, &nrem); + } + } + na = nla_next(na, &rem); + } + } + + return -ENOENT; +} + +static uint64_t cpu_total(const struct taskstats *stats) +{ + return (uint64_t)stats->ac_utime + (uint64_t)stats->ac_stime; +} + +static void print_stats(const char *label, const struct taskstats *stats) +{ + ksft_print_msg("%s: cpu_total=%llu nvcsw=%llu nivcsw=%llu\n", + label, (unsigned long long)cpu_total(stats), + (unsigned long long)stats->nvcsw, + (unsigned long long)stats->nivcsw); +} + +static void *worker_thread(void *arg) +{ + struct worker_ctx *ctx = arg; + + burn_cpu_for_ns(BUSY_NS); + + pthread_mutex_lock(&ctx->lock); + ctx->ready = true; + pthread_cond_broadcast(&ctx->cond); + while (!ctx->release) + pthread_cond_wait(&ctx->cond, &ctx->lock); + pthread_mutex_unlock(&ctx->lock); + + return NULL; +} + +int main(void) +{ + struct worker_ctx ctx = { + .lock = PTHREAD_MUTEX_INITIALIZER, + .cond = PTHREAD_COND_INITIALIZER, + }; + struct taskstats before, after; + pthread_t thread; + pid_t tgid = getpid(); + int family_id; + int fd; + int ret; + + ksft_print_header(); + ksft_set_plan(1); + + if (geteuid()) + ksft_exit_skip("taskstats_fill_stats_tgid needs root\n"); + + fd = netlink_open(); + if (fd < 0) + ksft_exit_skip("failed to open generic netlink socket: %s\n", + strerror(-fd)); + + family_id = get_family_id(fd, TASKSTATS_GENL_NAME); + if (family_id < 0) + ksft_exit_skip("taskstats generic netlink family unavailable: %s\n", + strerror(-family_id)); + + /* Create worker thread that burns 200ms of CPU */ + if (pthread_create(&thread, NULL, worker_thread, &ctx) != 0) + ksft_exit_fail_msg("pthread_create failed: %s\n", strerror(errno)); + + /* Wait for worker to finish generating activity */ + pthread_mutex_lock(&ctx.lock); + while (!ctx.ready) + pthread_cond_wait(&ctx.cond, &ctx.lock); + pthread_mutex_unlock(&ctx.lock); + + /* + * Snapshot A: TGID stats while worker is alive and sleeping. + * Contains main thread + worker contributions. + */ + ret = get_taskstats(fd, family_id, TASKSTATS_CMD_ATTR_TGID, tgid, &before); + if (ret) + ksft_exit_fail_msg("TGID query before exit failed: %s\n", + strerror(-ret)); + + /* Release worker so it can exit, then join (deterministic wait). + * + * Kernel exit path ordering guarantees: + * do_exit() + * taskstats_exit() -> fill_tgid_exit() (accumulates worker into signal->stats) + * exit_notify() (releases the thread) + * do_task_dead() -> __schedule() (wakes joiner) + * + * So pthread_join() returns only after fill_tgid_exit() has completed. + */ + pthread_mutex_lock(&ctx.lock); + ctx.release = true; + pthread_cond_broadcast(&ctx.cond); + pthread_mutex_unlock(&ctx.lock); + + pthread_join(thread, NULL); + + /* + * Snapshot B: TGID stats after worker has exited. + * fill_stats_for_tgid() does: + * memcpy(signal->stats) <- includes fill_tgid_exit accumulation + * + scan live threads <- only main thread now + */ + ret = get_taskstats(fd, family_id, TASKSTATS_CMD_ATTR_TGID, tgid, &after); + if (ret) + ksft_exit_fail_msg("TGID query after exit failed: %s\n", + strerror(-ret)); + + print_stats("TGID before worker exit", &before); + print_stats("TGID after worker exit", &after); + + /* + * The worker burned 200ms of CPU before the first snapshot. + * If the kernel correctly retained its contribution via + * fill_tgid_exit(), then the TGID CPU total after exit must be at + * least as large as the TGID CPU total before exit. + */ + ksft_test_result(cpu_total(&after) >= cpu_total(&before), + "TGID CPU stats should not regress after thread exit\n"); + + close(fd); + ksft_finished(); + return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS; +} |
