Skip to content

DAOS-18539 cart: detect SLINGSHOT_VNIS when set #17583

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
soumagne wants to merge 1 commit into
base: master
Choose a base branch
from
+56 −92
Open

DAOS-18539 cart: detect SLINGSHOT_VNIS when set #17583

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 40 additions & 42 deletions src/cart/crt_init.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,11 @@ dump_opt(crt_init_options_t *opt)
D_INFO("domain = %s\n", opt->cio_domain);
D_INFO("port = %s\n", opt->cio_port);
D_INFO("auth_key = %s\n", opt->cio_auth_key);
D_INFO("Flags: fault_inject = %d, use_credits = %d, use_sensors = %d, "
"thread_mode_single = %d, progress_busy = %d, mem_device = %d\n",
opt->cio_fault_inject, opt->cio_use_credits, opt->cio_use_sensors,
opt->cio_thread_mode_single, opt->cio_progress_busy, opt->cio_mem_device);
D_INFO("ep_credits = %d\n", opt->cio_ep_credits);
D_INFO("Flags: fault_inject = %d, use_sensors = %d, thread_mode_single = %d, "
"progress_busy = %d, mem_device = %d\n",
opt->cio_fault_inject, opt->cio_use_sensors, opt->cio_thread_mode_single,
opt->cio_progress_busy, opt->cio_mem_device);

if (opt->cio_use_expected_size)
D_INFO("max_expected_size = %d\n", opt->cio_max_expected_size);
Expand Down Expand Up @@ -160,10 +161,9 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider, bool p
crt_init_options_t *opt)

{
uint32_t ctx_num = 0;
uint32_t max_expect_size = 0;
uint32_t max_unexpect_size = 0;
uint32_t max_num_ctx = CRT_SRV_CONTEXT_NUM;
uint32_t ctx_max_num = 0;
int i;
int rc;

Expand All @@ -172,27 +172,29 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider, bool p
return rc;

if (crt_is_service()) {
ctx_num = CRT_SRV_CONTEXT_NUM;
max_num_ctx = CRT_SRV_CONTEXT_NUM;
ctx_max_num = CRT_SRV_CONTEXT_NUM;
} else {
/* Only limit the number of contexts for clients */
crt_env_get(CRT_CTX_NUM, &ctx_num);
CRT_ENV_OPT_GET(opt, ctx_max_num, CRT_CTX_NUM);

/* Default setting to the number of cores */
if (opt)
max_num_ctx =
ctx_num ? ctx_num : max(crt_gdata.cg_num_cores, opt->cio_ctx_max_num);
else
max_num_ctx = ctx_num ? ctx_num : crt_gdata.cg_num_cores;
}
if (!ctx_max_num)
ctx_max_num = crt_gdata.cg_num_cores;

if (max_num_ctx > CRT_SRV_CONTEXT_NUM)
max_num_ctx = CRT_SRV_CONTEXT_NUM;
/* To be able to run on VMs */
if (max_num_ctx < CRT_SRV_CONTEXT_NUM_MIN)
max_num_ctx = CRT_SRV_CONTEXT_NUM_MIN;
if (ctx_max_num > CRT_SRV_CONTEXT_NUM) {
D_WARN("ctx_max_num %u exceeds max %u, using max\n", ctx_max_num,
CRT_SRV_CONTEXT_NUM);
ctx_max_num = CRT_SRV_CONTEXT_NUM;
}
/* To be able to run on VMs */
if (ctx_max_num < CRT_SRV_CONTEXT_NUM_MIN) {
D_WARN("ctx_max_num %u is less than min %u, using min\n", ctx_max_num,
CRT_SRV_CONTEXT_NUM_MIN);
ctx_max_num = CRT_SRV_CONTEXT_NUM_MIN;
}
}

D_DEBUG(DB_ALL, "Max number of contexts set to %d\n", max_num_ctx);
D_DEBUG(DB_ALL, "Max number of contexts set to %u\n", ctx_max_num);

if (opt && opt->cio_use_expected_size)
max_expect_size = opt->cio_max_expected_size;
Expand All @@ -205,7 +207,7 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider, bool p
prov_data->cpg_ctx_num = 0;
prov_data->cpg_sep_mode = false;
prov_data->cpg_contig_ports = true;
prov_data->cpg_ctx_max_num = max_num_ctx;
prov_data->cpg_ctx_max_num = ctx_max_num;
prov_data->cpg_max_exp_size = max_expect_size;
prov_data->cpg_max_unexp_size = max_unexpect_size;
prov_data->cpg_primary = primary;
Expand All @@ -218,7 +220,7 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider, bool p
prov_data->cpg_last_remote_tag = 0;

D_DEBUG(DB_ALL, "prov_idx: %d primary: %d sizes: (%d/%d) max_ctx: %d\n", provider, primary,
max_expect_size, max_unexpect_size, max_num_ctx);
max_expect_size, max_unexpect_size, ctx_max_num);

D_INIT_LIST_HEAD(&prov_data->cpg_ctx_list);

Expand Down Expand Up @@ -261,8 +263,8 @@ crt_str_to_tc(const char *str)
static int
data_init(int server, crt_init_options_t *opt)
{
uint32_t timeout = 0;
uint32_t credits;
uint32_t crt_timeout = 0;
uint32_t ep_credits = CRT_DEFAULT_CREDITS_PER_EP_CTX;
uint32_t fi_univ_size = 0;
uint32_t mem_pin_enable = 0;
uint32_t is_secondary;
Expand Down Expand Up @@ -310,24 +312,20 @@ data_init(int server, crt_init_options_t *opt)
}
crt_gdata.cg_provider_is_primary = (is_secondary) ? 0 : 1;

if (opt && opt->cio_crt_timeout != 0)
timeout = opt->cio_crt_timeout;
else
crt_env_get(CRT_TIMEOUT, &timeout);
CRT_ENV_OPT_GET(opt, crt_timeout, CRT_TIMEOUT);
crt_gdata.cg_timeout =
(crt_timeout == 0 || crt_timeout > 3600) ? CRT_DEFAULT_TIMEOUT_S : crt_timeout;

if (timeout == 0 || timeout > 3600)
crt_gdata.cg_timeout = CRT_DEFAULT_TIMEOUT_S;
else
crt_gdata.cg_timeout = timeout;
crt_gdata.cg_swim_ctx_idx = CRT_DEFAULT_PROGRESS_CTX_IDX;

/* Override defaults and environment if option is set */
if (opt && opt->cio_use_credits) {
credits = opt->cio_ep_credits;
} else {
credits = CRT_DEFAULT_CREDITS_PER_EP_CTX;
crt_env_get(CRT_CREDIT_EP_CTX, &credits);
CRT_ENV_OPT_GET(opt, ep_credits, CRT_CREDIT_EP_CTX);
if (ep_credits > CRT_MAX_CREDITS_PER_EP_CTX) {
D_WARN("ep_credits %u exceeds max %u, using max\n", ep_credits,
CRT_MAX_CREDITS_PER_EP_CTX);
ep_credits = CRT_MAX_CREDITS_PER_EP_CTX;
}
crt_gdata.cg_credit_ep_ctx = ep_credits;

/* Enable quotas by default only on clients */
crt_gdata.cg_rpc_quota = server ? 0 : CRT_QUOTA_RPCS_DEFAULT;
Expand All @@ -352,10 +350,6 @@ data_init(int server, crt_init_options_t *opt)
d_setenv("FI_UNIVERSE_SIZE", "2048", 1);
}

if (credits > CRT_MAX_CREDITS_PER_EP_CTX)
credits = CRT_MAX_CREDITS_PER_EP_CTX;
crt_gdata.cg_credit_ep_ctx = credits;

/** enable sensors if requested */
crt_gdata.cg_use_sensors = (opt && opt->cio_use_sensors);

Expand Down Expand Up @@ -752,6 +746,10 @@ crt_init_opt(crt_group_id_t grpid, uint32_t flags, crt_init_options_t *opt)
if (interface == NULL && prov != CRT_PROV_OFI_CXI)
D_WARN("No interface specified\n");

/* For PALS-enabled environments, auto-detect svc ID / VNI and use DAOS VNI */
if (prov == CRT_PROV_OFI_CXI && auth_key == NULL && getenv("SLINGSHOT_VNIS") != NULL)
auth_key = "0:0:2"; /* format is svc_id:vni:vni_idx */

crt_gdata.cg_primary_prov = prov;
/*
* Note: If on the client the 'interface' contains a
Expand Down
28 changes: 3 additions & 25 deletions src/cart/crt_internal_types.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ struct crt_event_cb_priv {
/*
* List of environment variables to read at CaRT library load time.
* for integer envs use ENV()
* for string ones ENV_STR() or ENV_STR_NO_PRINT()
* for string ones ENV_STR()
**/
#define CRT_ENV_LIST \
ENV_STR(CRT_ATTACH_INFO_PATH) \
Expand Down Expand Up @@ -243,7 +243,8 @@ struct crt_event_cb_priv {
ENV(D_MRECV_BUF) \
ENV(D_MRECV_BUF_COPY) \
ENV_STR(D_PROVIDER) \
ENV_STR_NO_PRINT(D_PROVIDER_AUTH_KEY) \
ENV_STR(D_PROVIDER_AUTH_KEY) \
ENV_STR(SLINGSHOT_VNIS) \
ENV(D_QUOTA_RPCS) \
ENV(D_QUOTA_BULKS) \
ENV(FI_OFI_RXM_USE_SRX) \
Expand All @@ -267,16 +268,13 @@ struct crt_event_cb_priv {
int _rc_##x; \
bool _no_print_##x;

#define ENV_STR_NO_PRINT(x) ENV_STR(x)

struct crt_envs {
CRT_ENV_LIST;
bool inited;
};

#undef ENV
#undef ENV_STR
#undef ENV_STR_NO_PRINT

extern struct crt_envs crt_genvs;

Expand All @@ -303,16 +301,9 @@ crt_env_init(void)
crt_genvs._no_print_##x = false; \
} while (0);

#define ENV_STR_NO_PRINT(x) \
do { \
crt_genvs._rc_##x = d_agetenv_str(&crt_genvs._##x, #x); \
crt_genvs._no_print_##x = true; \
} while (0);

CRT_ENV_LIST;
#undef ENV
#undef ENV_STR
#undef ENV_STR_NO_PRINT

crt_genvs.inited = true;
}
Expand All @@ -323,13 +314,11 @@ crt_env_fini(void)
{
#define ENV(x) (void)
#define ENV_STR(x) d_freeenv_str(&crt_genvs._##x);
#define ENV_STR_NO_PRINT ENV_STR

CRT_ENV_LIST

#undef ENV
#undef ENV_STR
#undef ENV_STR_NO_PRINT

crt_genvs.inited = false;
}
Expand Down Expand Up @@ -357,20 +346,12 @@ crt_env_list_valid(void)
return false; \
}

/* if string env exceeds CRT_ENV_STR_MAX_SIZE - return false */
#define ENV_STR_NO_PRINT(x) \
if (crt_genvs._rc_##x == 0 && strlen(crt_genvs._##x) + 1 > CRT_ENV_STR_MAX_SIZE) { \
D_ERROR("env '%s' exceeded max size %d\n", #x, CRT_ENV_STR_MAX_SIZE); \
return false; \
}

/* expand env list using the above ENV_* definitions */
CRT_ENV_LIST;
return true;

#undef ENV
#undef ENV_STR
#undef ENV_STR_NO_PRINT
}

/* dump environment variables from the CRT_ENV_LIST */
Expand All @@ -388,13 +369,10 @@ crt_env_dump(void)
if (!crt_genvs._rc_##x) \
D_INFO("%s = %s\n", #x, crt_genvs._no_print_##x ? "****" : crt_genvs._##x);

#define ENV_STR_NO_PRINT ENV_STR

CRT_ENV_LIST;

#undef ENV
#undef ENV_STR
#undef ENV_STR_NO_PRINT
}

/* structure of global fault tolerance data */
Expand Down
5 changes: 0 additions & 5 deletions src/control/server/config/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,6 @@ func (cfg *Server) WithFabricProvider(provider string) *Server {
// WithFabricAuthKey sets the top-level fabric authorization key.
func (cfg *Server) WithFabricAuthKey(key string) *Server {
cfg.Fabric.AuthKey = key
cfg.ClientEnvVars = common.MergeKeyValues(cfg.ClientEnvVars, []string{cfg.Fabric.GetAuthKeyEnv()})
for _, engine := range cfg.Engines {
engine.Fabric.AuthKey = cfg.Fabric.AuthKey
}
Expand Down Expand Up @@ -404,10 +403,6 @@ func (cfg *Server) Load(log logging.Logger) error {
cfg.updateServerConfig(&cfg.Engines[i])
}

if cfg.Fabric.AuthKey != "" {
cfg.ClientEnvVars = common.MergeKeyValues(cfg.ClientEnvVars, []string{cfg.Fabric.GetAuthKeyEnv()})
}

if len(cfg.deprecatedParams.AccessPoints) > 0 {
if len(cfg.MgmtSvcReplicas) > 0 {
return errors.New(msgAPsMSReps)
Expand Down
26 changes: 10 additions & 16 deletions src/include/cart/types.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,26 +46,20 @@ typedef struct crt_init_options {
* evnironment variable.
*/
int cio_crt_timeout;
uint32_t cio_sep_override:1, /**< Deprecated */
cio_use_sep:1, /**< Deprecated */
/** whether or not to inject faults */
cio_fault_inject:1,
/**
* whether or not to override credits. When set
* overrides CRT_CTX_EP_CREDITS envariable
*/
cio_use_credits:1,
/** whether or not to enable per-context sensors */
cio_use_sensors:1,

/** whether or not to use expected sizes */
cio_use_expected_size:1,
cio_use_unexpected_size:1;
uint32_t cio_sep_override : 1, /**< Deprecated */
cio_use_sep : 1, /**< Deprecated */
/** whether or not to inject faults */
cio_fault_inject : 1,
/** whether or not to enable per-context sensors */
cio_use_sensors : 1,

/** whether or not to use expected sizes */
cio_use_expected_size : 1, cio_use_unexpected_size : 1;

/** overrides the value of the environment variable CRT_CTX_NUM */
int cio_ctx_max_num;

/** Used with cio_use_credits to set credit limit */
/** set credit limit */
int cio_ep_credits;

/**
Expand Down
2 changes: 1 addition & 1 deletion src/tests/ftest/cart/test_ep_cred_client.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/*
* (C) Copyright 2018-2022 Intel Corporation.
* (C) Copyright 2026 Hewlett Packard Enterprise Development LP
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
Expand Down Expand Up @@ -61,7 +62,6 @@ test_run()
D_ASSERTF(rc == 0, "crt_group_config_path_set failed %d\n", rc);
}

opt.cio_use_credits = 1;
opt.cio_ep_credits = test.tg_credits;

DBG_PRINT("Number of credits: %d Number of burst: %d\n",
Expand Down
3 changes: 1 addition & 2 deletions src/tests/ftest/cart/test_ep_cred_server.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
* (C) Copyright 2018-2022 Intel Corporation.
* (C) Copyright 2025 Hewlett Packard Enterprise Development LP
* (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
Expand All @@ -21,7 +21,6 @@ test_run(d_rank_t my_rank)
DBG_PRINT("local group: %s remote group: %s\n",
test.tg_local_group_name, test.tg_remote_group_name);

opt.cio_use_credits = 1;
opt.cio_ep_credits = test.tg_credits;

rc = crtu_srv_start_basic(test.tg_local_group_name, &test.tg_crt_ctx, &test.tg_tid, &grp,
Expand Down
2 changes: 1 addition & 1 deletion utils/config/daos_server.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@
#
## CART: Fabric authorization key
## If the fabric requires an authorization key, set it here to
## be used on the server and clients.
## be used on the server.
#
#fabric_auth_key: foo:bar
#
Expand Down
Loading