Viewing: tgt_grant.c
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
* Use is subject to license terms.
*
* Copyright (c) 2012, 2017, Intel Corporation.
*/
/*
* This file provides code related to grant space management on Lustre Targets
* (OSTs and MDTs). Grant is a mechanism used by client nodes to reserve disk
* space on a target for the data writeback cache. The Lustre client is thus
* assured that enough space will be available when flushing dirty pages
* asynchronously. Each client node is granted an initial amount of reserved
* space at connect time and gets additional space back from target in bulk
* write reply.
*
* We actually support three different cases:
* - The client supports the new grant parameters (i.e. OBD_CONNECT_GRANT_PARAM)
* which means that all grant overhead calculation happens on the client side.
* The server reports at connect time the backend filesystem block size, the
* maximum extent size as well as the extent insertion cost and it is then up
* to the osc layer to the track dirty extents and consume grant accordingly
* (see osc_cache.c). In each bulk write request, the client provides how much
* grant space was consumed for this RPC.
* - The client does not support OBD_CONNECT_GRANT_PARAM and always assumes a
* a backend file system block size of 4KB. We then have two cases:
* - If the block size is really 4KB, then the client can deal with grant
* allocation for partial block writes, but won't take extent insertion cost
* into account. For such clients, we inflate grant by 100% on the server
* side. It means that when 32MB of grant is hold by the client, 64MB of
* grant space is actually reserved on the server. All grant counters
* provided by such a client are inflated by 100%.
* - The backend filesystem block size is bigger than 4KB, which isn't
* supported by the client. In this case, we emulate a 4KB block size and
* consume one block size on the server for each 4KB of grant returned to
* client. With a 128KB blocksize, it means that 32MB dirty pages of 4KB
* on the client will actually consume 1GB of grant on the server.
* All grant counters provided by such a client are inflated by the block
* size ratio.
*
* This file handles the core logic for:
* - grant allocation strategy
* - maintaining per-client as well as global grant space accounting
* - processing grant information packed in incoming requests
* - allocating server-side grant space for synchronous write RPCs which did not
* consume grant on the client side (OBD_BRW_FROM_GRANT flag not set). If not
* enough space is available, such RPCs fail with ENOSPC
*
* Author: Johann Lombardi <johann.lombardi@intel.com>
*/
#define DEBUG_SUBSYSTEM S_CLASS
#include <obd.h>
#include <obd_class.h>
#include "tgt_internal.h"
static int lbug_on_grant_miscount;
module_param(lbug_on_grant_miscount, int, 0644);
MODULE_PARM_DESC(lbug_on_grant_miscount, "LBUG on grant miscount");
/* Helpers to inflate/deflate grants for clients that do not support the grant
* parameters */
static inline u64 tgt_grant_inflate(struct tg_grants_data *tgd, u64 val)
{
if (tgd->tgd_blockbits > COMPAT_BSIZE_SHIFT)
/* Client does not support such large block size, grant
* is thus inflated. We already significantly overestimate
* overhead, no need to add the extent tax in this case */
return val << (tgd->tgd_blockbits - COMPAT_BSIZE_SHIFT);
return val;
}
/* Companion of tgt_grant_inflate() */
static inline u64 tgt_grant_deflate(struct tg_grants_data *tgd, u64 val)
{
if (tgd->tgd_blockbits > COMPAT_BSIZE_SHIFT)
return val >> (tgd->tgd_blockbits - COMPAT_BSIZE_SHIFT);
return val;
}
/* Grant chunk is used as a unit for grant allocation. It should be inflated
* if the client does not support the grant paramaters.
* Check connection flag against \a data if not NULL. This is used during
* connection creation where exp->exp_connect_data isn't populated yet */
static inline u64 tgt_grant_chunk(struct obd_export *exp,
struct lu_target *lut,
struct obd_connect_data *data)
{
struct tg_grants_data *tgd = &lut->lut_tgd;
u64 chunk = exp_max_brw_size(exp);
u64 tax;
if (exp->exp_obd->obd_self_export == exp)
/* Grant enough space to handle a big precreate request */
return OST_MAX_PRECREATE * lut->lut_dt_conf.ddp_inodespace / 2;
if ((data == NULL && !(exp_grant_param_supp(exp))) ||
(data != NULL && !OCD_HAS_FLAG(data, GRANT_PARAM)))
/* Try to grant enough space to send 2 full-size RPCs */
return tgt_grant_inflate(tgd, chunk) << 1;
/* Try to return enough to send two full-size RPCs
* = 2 * (BRW_size + #extents_in_BRW * grant_tax) */
tax = 1ULL << tgd->tgd_blockbits; /* block size */
tax *= lut->lut_dt_conf.ddp_max_extent_blks; /* max extent size */
tax = (chunk + tax - 1) / tax; /* #extents in a RPC */
tax *= lut->lut_dt_conf.ddp_extent_tax; /* extent tax for a RPC */
chunk = (chunk + tax) * 2; /* we said two full RPCs */
return chunk;
}
static int tgt_check_export_grants(struct obd_export *exp, u64 *dirty,
u64 *pending, u64 *granted, u64 maxsize)
{
struct tg_export_data *ted = &exp->exp_target_data;
int level = D_CACHE;
if (ted->ted_grant < 0 || ted->ted_pending < 0 || ted->ted_dirty < 0)
level = D_ERROR;
CDEBUG_LIMIT(level, "%s: cli %s/%p dirty %ld pend %ld grant %ld\n",
exp->exp_obd->obd_name, exp->exp_client_uuid.uuid, exp,
ted->ted_dirty, ted->ted_pending, ted->ted_grant);
if (ted->ted_grant + ted->ted_pending > maxsize) {
CERROR("%s: cli %s/%p ted_grant(%ld) + ted_pending(%ld) > maxsize(%llu)\n",
exp->exp_obd->obd_name,
exp->exp_client_uuid.uuid, exp, ted->ted_grant,
ted->ted_pending, maxsize);
return -EFAULT;
}
if (ted->ted_dirty > maxsize) {
CERROR("%s: cli %s/%p ted_dirty(%ld) > maxsize(%llu)\n",
exp->exp_obd->obd_name, exp->exp_client_uuid.uuid,
exp, ted->ted_dirty, maxsize);
return -EFAULT;
}
*granted += ted->ted_grant + ted->ted_pending;
*pending += ted->ted_pending;
*dirty += ted->ted_dirty;
return 0;
}
/**
* Perform extra sanity checks for grant accounting.
*
* This function scans the export list, sanity checks per-export grant counters
* and verifies accuracy of global grant accounting. If an inconsistency is
* found, a CERROR is printed with the function name \func that was passed as
* argument. LBUG is only called in case of serious counter corruption (i.e.
* value larger than the device size).
* Those sanity checks can be pretty expensive and are disabled if the OBD
* device has more than 100 connected exports by default.
*
* \param[in] obd OBD device for which grant accounting should be
* verified
* \param[in] func caller's function name
*/
void tgt_grant_sanity_check(struct obd_device *obd, const char *func)
{
struct lu_target *lut = obd2obt(obd)->obt_lut;
struct tg_grants_data *tgd = &lut->lut_tgd;
struct obd_export *exp;
struct tg_export_data *ted;
u64 maxsize;
u64 tot_dirty = 0;
u64 tot_pending = 0;
u64 tot_granted = 0;
u64 fo_tot_granted;
u64 fo_tot_pending;
u64 fo_tot_dirty;
int error;
if (list_empty(&obd->obd_exports))
return;
/*
* We don't want to do this for large machines that do lots of
* mounts or unmounts. It burns...
* Use set_param to change obd_grant_check_threshold, which
* is 100 by default, 0 to always check grants
*/
if (obd->obd_num_exports > obd->obd_grant_check_threshold &&
obd->obd_grant_check_threshold)
return;
maxsize = tgd->tgd_osfs.os_blocks << tgd->tgd_blockbits;
spin_lock(&obd->obd_dev_lock);
spin_lock(&tgd->tgd_grant_lock);
exp = obd->obd_self_export;
ted = &exp->exp_target_data;
CDEBUG(D_CACHE, "%s: processing self export: %ld %ld %ld\n",
obd->obd_name, ted->ted_grant,
ted->ted_pending, ted->ted_dirty);
tot_granted += ted->ted_grant + ted->ted_pending;
tot_pending += ted->ted_pending;
tot_dirty += ted->ted_dirty;
list_for_each_entry(exp, &obd->obd_exports, exp_obd_chain) {
error = tgt_check_export_grants(exp, &tot_dirty, &tot_pending,
&tot_granted, maxsize);
if (error < 0) {
spin_unlock(&obd->obd_dev_lock);
spin_unlock(&tgd->tgd_grant_lock);
LBUG();
}
}
/* exports about to be unlinked should also be taken into account since
* they might still hold pending grant space to be released at
* commit time */
list_for_each_entry(exp, &obd->obd_unlinked_exports, exp_obd_chain) {
error = tgt_check_export_grants(exp, &tot_dirty, &tot_pending,
&tot_granted, maxsize);
if (error < 0) {
spin_unlock(&obd->obd_dev_lock);
spin_unlock(&tgd->tgd_grant_lock);
LBUG();
}
}
fo_tot_granted = tgd->tgd_tot_granted;
fo_tot_pending = tgd->tgd_tot_pending;
fo_tot_dirty = tgd->tgd_tot_dirty;
spin_unlock(&obd->obd_dev_lock);
spin_unlock(&tgd->tgd_grant_lock);
if (tot_granted != fo_tot_granted)
CERROR("%s: tot_granted %llu != fo_tot_granted %llu\n",
func, tot_granted, fo_tot_granted);
if (tot_pending != fo_tot_pending)
CERROR("%s: tot_pending %llu != fo_tot_pending %llu\n",
func, tot_pending, fo_tot_pending);
if (tot_dirty != fo_tot_dirty)
CERROR("%s: tot_dirty %llu != fo_tot_dirty %llu\n",
func, tot_dirty, fo_tot_dirty);
if (tot_pending > tot_granted)
CERROR("%s: tot_pending %llu > tot_granted %llu\n",
func, tot_pending, tot_granted);
if (tot_granted > maxsize)
CERROR("%s: tot_granted %llu > maxsize %llu\n",
func, tot_granted, maxsize);
if (tot_dirty > maxsize)
CERROR("%s: tot_dirty %llu > maxsize %llu\n",
func, tot_dirty, maxsize);
}
EXPORT_SYMBOL(tgt_grant_sanity_check);
/**
* Get file system statistics of target.
*
* Helper function for statfs(), also used by grant code.
* Implements caching for statistics to avoid calling OSD device each time.
*
* \param[in] env execution environment
* \param[in] lut LU target
* \param[out] osfs statistic data to return
* \param[in] max_age maximum age for cached data
* \param[in] from_cache show that data was get from cache or not
*
* \retval 0 if successful
* \retval negative value on error
*/
int tgt_statfs_internal(const struct lu_env *env, struct lu_target *lut,
struct obd_statfs *osfs, time64_t max_age, int *from_cache)
{
struct tg_grants_data *tgd = &lut->lut_tgd;
int rc = 0;
ENTRY;
spin_lock(&tgd->tgd_osfs_lock);
if (tgd->tgd_osfs_age < max_age || max_age == 0) {
u64 unstable;
/* statfs data are too old, get up-to-date one.
* we must be cautious here since multiple threads might be
* willing to update statfs data concurrently and we must
* grant that cached statfs data are always consistent */
if (tgd->tgd_statfs_inflight == 0)
/* clear inflight counter if no users, although it would
* take a while to overflow this 64-bit counter ... */
tgd->tgd_osfs_inflight = 0;
/* notify tgt_grant_commit() that we want to track writes
* completed as of now */
tgd->tgd_statfs_inflight++;
/* record value of inflight counter before running statfs to
* compute the diff once statfs is completed */
unstable = tgd->tgd_osfs_inflight;
spin_unlock(&tgd->tgd_osfs_lock);
/* statfs can sleep ... hopefully not for too long since we can
* call it fairly often as space fills up */
rc = dt_statfs(env, lut->lut_bottom, osfs);
if (unlikely(rc))
GOTO(out, rc);
osfs->os_namelen = min_t(__u32, osfs->os_namelen, NAME_MAX);
spin_lock(&tgd->tgd_grant_lock);
spin_lock(&tgd->tgd_osfs_lock);
/* calculate how much space was written while we released the
* tgd_osfs_lock */
unstable = tgd->tgd_osfs_inflight - unstable;
tgd->tgd_osfs_unstable = 0;
if (unstable) {
/* some writes committed while we were running statfs
* w/o the tgd_osfs_lock. Those ones got added to
* the cached statfs data that we are about to crunch.
* Take them into account in the new statfs data */
osfs->os_bavail -= min_t(u64, osfs->os_bavail,
unstable >> tgd->tgd_blockbits);
/* However, we don't really know if those writes got
* accounted in the statfs call, so tell
* tgt_grant_space_left() there is some uncertainty
* on the accounting of those writes.
* The purpose is to prevent spurious error messages in
* tgt_grant_space_left() since those writes might be
* accounted twice. */
tgd->tgd_osfs_unstable += unstable;
}
/* similarly, there is some uncertainty on write requests
* between prepare & commit */
tgd->tgd_osfs_unstable += tgd->tgd_tot_pending;
spin_unlock(&tgd->tgd_grant_lock);
/* finally udpate cached statfs data */
tgd->tgd_osfs = *osfs;
tgd->tgd_osfs_age = ktime_get_seconds();
tgd->tgd_statfs_inflight--; /* stop tracking */
if (tgd->tgd_statfs_inflight == 0)
tgd->tgd_osfs_inflight = 0;
spin_unlock(&tgd->tgd_osfs_lock);
if (from_cache)
*from_cache = 0;
} else {
/* use cached statfs data */
*osfs = tgd->tgd_osfs;
spin_unlock(&tgd->tgd_osfs_lock);
if (from_cache)
*from_cache = 1;
}
GOTO(out, rc);
out:
return rc;
}
EXPORT_SYMBOL(tgt_statfs_internal);
/**
* Update cached statfs information from the OSD layer
*
* Refresh statfs information cached in tgd::tgd_osfs if the cache is older
* than 1s or if force is set. The OSD layer is in charge of estimating data &
* metadata overhead.
* This function can sleep so it should not be called with any spinlock held.
*
* \param[in] env LU environment passed by the caller
* \param[in] exp export used to print client info in debug
* messages
* \param[in] force force a refresh of statfs information
* \param[out] from_cache returns whether the statfs information are
* taken from cache
*/
static void tgt_grant_statfs(const struct lu_env *env, struct obd_export *exp,
int force, int *from_cache)
{
struct obd_device *obd = exp->exp_obd;
struct lu_target *lut = obd2obt(obd)->obt_lut;
struct tg_grants_data *tgd = &lut->lut_tgd;
struct tgt_thread_info *tti;
struct obd_statfs *osfs;
time64_t max_age;
int rc;
if (force)
max_age = 0; /* get fresh statfs data */
else
max_age = ktime_get_seconds() - OBD_STATFS_CACHE_SECONDS;
tti = tgt_th_info(env);
osfs = &tti->tti_u.osfs;
rc = tgt_statfs_internal(env, lut, osfs, max_age, from_cache);
if (unlikely(rc)) {
if (from_cache)
*from_cache = 0;
return;
}
CDEBUG(D_CACHE, "%s: cli %s/%p free: %llu avail: %llu\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp,
osfs->os_bfree << tgd->tgd_blockbits,
osfs->os_bavail << tgd->tgd_blockbits);
}
/**
* Figure out how much space is available on the backend filesystem after
* removing grant space already booked by clients.
*
* This is done by accessing cached statfs data previously populated by
* tgt_grant_statfs(), from which we withdraw the space already granted to
* clients and the reserved space.
* Caller must hold tgd_grant_lock spinlock.
*
* \param[in] exp export associated with the device for which the amount
* of available space is requested
* \retval amount of non-allocated space, in bytes
*/
static u64 tgt_grant_space_left(struct obd_export *exp)
{
struct obd_device *obd = exp->exp_obd;
struct lu_target *lut = obd2obt(obd)->obt_lut;
struct tg_grants_data *tgd = &lut->lut_tgd;
u64 tot_granted;
u64 left;
u64 avail;
u64 unstable;
u64 reserved;
ENTRY;
assert_spin_locked(&tgd->tgd_grant_lock);
spin_lock(&tgd->tgd_osfs_lock);
/* get available space from cached statfs data */
left = tgd->tgd_osfs.os_bavail << tgd->tgd_blockbits;
unstable = tgd->tgd_osfs_unstable; /* those might be accounted twice */
spin_unlock(&tgd->tgd_osfs_lock);
reserved = left * tgd->tgd_reserved_pcnt / 100;
tot_granted = tgd->tgd_tot_granted + reserved;
if (left < tot_granted) {
int mask = (left + unstable <
tot_granted - tgd->tgd_tot_pending) ?
D_ERROR : D_CACHE;
/* the below message is checked in sanityn.sh test_15 */
CDEBUG_LIMIT(mask,
"%s: cli %s/%p left=%llu < tot_grant=%llu unstable=%llu pending=%llu dirty=%llu\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp,
left, tot_granted, unstable,
tgd->tgd_tot_pending,
tgd->tgd_tot_dirty);
RETURN(0);
}
avail = left;
/* Withdraw space already granted to clients */
left -= tot_granted;
/* Align left on block size */
left &= ~((1ULL << tgd->tgd_blockbits) - 1);
CDEBUG(D_CACHE,
"%s: cli %s/%p avail=%llu left=%llu unstable=%llu tot_grant=%llu pending=%llu\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp, avail, left,
unstable, tot_granted, tgd->tgd_tot_pending);
RETURN(left);
}
/**
* Process grant information from obdo structure packed in incoming BRW
* and inflate grant counters if required.
*
* Grab the dirty and seen grant announcements from the incoming obdo and
* inflate all grant counters passed in the request if the client does not
* support the grant parameters.
* We will later calculate the client's new grant and return it.
* Caller must hold tgd_grant_lock spinlock.
*
* \param[in] env LU environment supplying osfs storage
* \param[in] exp export for which we received the request
* \param[in,out] oa incoming obdo sent by the client
*/
static void tgt_grant_incoming(const struct lu_env *env, struct obd_export *exp,
struct obdo *oa, long chunk)
{
struct tg_export_data *ted = &exp->exp_target_data;
struct obd_device *obd = exp->exp_obd;
struct tg_grants_data *tgd = &obd2obt(obd)->obt_lut->lut_tgd;
long long dirty, dropped;
ENTRY;
assert_spin_locked(&tgd->tgd_grant_lock);
if ((oa->o_valid & (OBD_MD_FLBLOCKS|OBD_MD_FLGRANT)) !=
(OBD_MD_FLBLOCKS|OBD_MD_FLGRANT)) {
oa->o_valid &= ~OBD_MD_FLGRANT;
RETURN_EXIT;
}
/* Add some margin, since there is a small race if other RPCs arrive
* out-or-order and have already consumed some grant. We want to
* leave this here in case there is a large error in accounting. */
CDEBUG(D_CACHE,
"%s: cli %s/%p reports grant %llu dropped %u, local %lu\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp, oa->o_grant,
oa->o_dropped, ted->ted_grant);
if ((long long)oa->o_dirty < 0)
oa->o_dirty = 0;
/* inflate grant counters if required */
if (!exp_grant_param_supp(exp)) {
u64 tmp;
oa->o_grant = tgt_grant_inflate(tgd, oa->o_grant);
oa->o_dirty = tgt_grant_inflate(tgd, oa->o_dirty);
/* inflation can bump client's wish to >4GB which doesn't fit
* 32bit o_undirty, limit that .. */
tmp = tgt_grant_inflate(tgd, oa->o_undirty);
if (tmp >= OBD_MAX_GRANT)
tmp = OBD_MAX_GRANT & ~(1ULL << tgd->tgd_blockbits);
oa->o_undirty = tmp;
tmp = tgt_grant_inflate(tgd, oa->o_dropped);
if (tmp >= OBD_MAX_GRANT)
tmp = OBD_MAX_GRANT & ~(1ULL << tgd->tgd_blockbits);
oa->o_dropped = tmp;
}
dirty = oa->o_dirty;
dropped = oa->o_dropped;
/* Update our accounting now so that statfs takes it into account.
* Note that ted_dirty is only approximate and can become incorrect
* if RPCs arrive out-of-order. No important calculations depend
* on ted_dirty however, but we must check sanity to not assert. */
if (dirty > ted->ted_grant + 4 * chunk)
dirty = ted->ted_grant + 4 * chunk;
tgd->tgd_tot_dirty += dirty - ted->ted_dirty;
if (ted->ted_grant < dropped) {
CDEBUG(D_CACHE,
"%s: cli %s/%p reports %llu dropped > grant %lu\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp, dropped,
ted->ted_grant);
dropped = 0;
}
if (tgd->tgd_tot_granted < dropped) {
CERROR("%s: cli %s/%p reports %llu dropped > tot_grant %llu\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp,
dropped, tgd->tgd_tot_granted);
dropped = 0;
}
tgd->tgd_tot_granted -= dropped;
ted->ted_grant -= dropped;
ted->ted_dirty = dirty;
if (ted->ted_dirty < 0 || ted->ted_grant < 0 || ted->ted_pending < 0) {
CERROR("%s: cli %s/%p dirty %ld pend %ld grant %ld\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp,
ted->ted_dirty, ted->ted_pending, ted->ted_grant);
spin_unlock(&tgd->tgd_grant_lock);
LBUG();
}
EXIT;
}
/**
* Grant shrink request handler.
*
* Client nodes can explicitly release grant space (i.e. process called grant
* shrinking). This function proceeds with the shrink request when there is
* less ungranted space remaining than the amount all of the connected clients
* would consume if they used their full grant.
* Caller must hold tgd_grant_lock spinlock.
*
* \param[in] exp export releasing grant space
* \param[in,out] oa incoming obdo sent by the client
* \param[in] left_space remaining free space with space already granted
* taken out
*/
static void tgt_grant_shrink(struct obd_export *exp, struct obdo *oa,
u64 left_space)
{
struct tg_export_data *ted = &exp->exp_target_data;
struct obd_device *obd = exp->exp_obd;
struct tg_grants_data *tgd = &obd2obt(obd)->obt_lut->lut_tgd;
long grant_shrink;
assert_spin_locked(&tgd->tgd_grant_lock);
LASSERT(exp);
/* don't need to shrink grant if it uses less than 25% of left space */
if (tgd->tgd_tot_granted * 4 < left_space)
return;
grant_shrink = oa->o_grant;
if (ted->ted_grant < grant_shrink) {
CDEBUG(D_CACHE,
"%s: cli %s/%p wants %lu shrinked > grant %lu\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp,
grant_shrink, ted->ted_grant);
grant_shrink = ted->ted_grant;
}
ted->ted_grant -= grant_shrink;
tgd->tgd_tot_granted -= grant_shrink;
CDEBUG(D_CACHE, "%s: cli %s/%p shrink %ld ted_grant %ld total %llu\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp, grant_shrink,
ted->ted_grant, tgd->tgd_tot_granted);
/* client has just released some grant, don't grant any space back */
oa->o_grant = 0;
}
/**
* Calculate how much space is required to write a given network buffer
*
* This function takes block alignment into account to estimate how much on-disk
* space will be required to successfully write the whole niobuf.
* Estimated space is inflated if the export does not support
* OBD_CONNECT_GRANT_PARAM and if the backend filesystem has a block size
* larger than the minimal supported page size (i.e. 4KB).
*
* \param[in] exp export associated which the write request
* if NULL, then size estimate is done for server-side
* grant allocation.
* \param[in] lut LU target handling the request
* \param[in] rnb network buffer to estimate size of
*
* \retval space (in bytes) that will be consumed to write the
* network buffer
*/
static inline u64 tgt_grant_rnb_size(struct obd_export *exp,
struct lu_target *lut,
struct niobuf_remote *rnb)
{
struct tg_grants_data *tgd = &lut->lut_tgd;
u64 blksize;
u64 bytes;
u64 end;
if (exp && !exp_grant_param_supp(exp) &&
tgd->tgd_blockbits > COMPAT_BSIZE_SHIFT)
blksize = 1ULL << COMPAT_BSIZE_SHIFT;
else
blksize = 1ULL << tgd->tgd_blockbits;
/* The network buffer might span several blocks, align it on block
* boundaries */
bytes = rnb->rnb_offset & (blksize - 1);
bytes += rnb->rnb_len;
end = bytes & (blksize - 1);
if (end)
bytes += blksize - end;
if (exp == NULL || exp_grant_param_supp(exp)) {
/* add per-extent insertion cost */
u64 max_ext;
int nr_ext;
max_ext = blksize * lut->lut_dt_conf.ddp_max_extent_blks;
nr_ext = (bytes + max_ext - 1) / max_ext;
bytes += nr_ext * lut->lut_dt_conf.ddp_extent_tax;
} else {
/* Inflate grant space if client does not support extent-based
* grant allocation */
bytes = tgt_grant_inflate(tgd, (u64)bytes);
}
return bytes;
}
/**
* Validate grant accounting for each incoming remote network buffer.
*
* When clients have dirtied as much space as they've been granted they
* fall through to sync writes. These sync writes haven't been expressed
* in grants and need to error with ENOSPC when there isn't room in the
* filesystem for them after grants are taken into account. However,
* writeback of the dirty data that was already granted space can write
* right on through.
* The OBD_BRW_GRANTED flag will be set in the rnb_flags of each network
* buffer which has been granted enough space to proceed. Buffers without
* this flag will fail to be written with -ENOSPC (see tgt_preprw_write().
* Caller must hold tgd_grant_lock spinlock.
*
* \param[in] env LU environment passed by the caller
* \param[in] exp export identifying the client which sent the RPC
* \param[in] oa incoming obdo in which we should return the pack the
* additional grant
* \param[in,out] rnb the list of network buffers
* \param[in] niocount the number of network buffers in the list
* \param[in] left the remaining free space with space already granted
* taken out
*/
static void tgt_grant_check(const struct lu_env *env, struct obd_export *exp,
struct obdo *oa, struct niobuf_remote *rnb,
int niocount, u64 *left)
{
struct tg_export_data *ted = &exp->exp_target_data;
struct obd_device *obd = exp->exp_obd;
struct lu_target *lut = obd2obt(obd)->obt_lut;
struct tg_grants_data *tgd = &lut->lut_tgd;
unsigned long ungranted = 0;
unsigned long granted = 0;
int i;
bool skip = false;
ENTRY;
assert_spin_locked(&tgd->tgd_grant_lock);
if (test_bit(OBDF_RECOVERING, obd->obd_flags)) {
/* Replaying write. Grant info have been processed already so no
* need to do any enforcement here. It is worth noting that only
* bulk writes with all rnbs having OBD_BRW_FROM_GRANT can be
* replayed. If one page hasn't OBD_BRW_FROM_GRANT set, then
* the whole bulk is written synchronously */
skip = true;
CDEBUG(D_CACHE, "Replaying write, skipping accounting\n");
} else if ((oa->o_valid & OBD_MD_FLFLAGS) &&
(oa->o_flags & OBD_FL_RECOV_RESEND)) {
/* Recoverable resend, grant info have already been processed as
* well */
skip = true;
CDEBUG(D_CACHE, "Recoverable resend arrived, skipping accounting\n");
} else if (exp_grant_param_supp(exp) && oa->o_grant_used > 0) {
/* Client supports the new grant parameters and is telling us
* how much grant space it consumed for this bulk write.
* Although all rnbs are supposed to have the OBD_BRW_FROM_GRANT
* flag set, we will scan the rnb list and looks for non-cache
* I/O in case it changes in the future */
if (ted->ted_grant >= oa->o_grant_used) {
/* skip grant accounting for rnbs with
* OBD_BRW_FROM_GRANT and just used grant consumption
* claimed in the request */
granted = oa->o_grant_used;
skip = true;
} else {
/* client has used more grants for this request that
* it owns ... */
CERROR("%s: cli %s claims %lu GRANT, real grant %lu\n",
exp->exp_obd->obd_name,
exp->exp_client_uuid.uuid,
(unsigned long)oa->o_grant_used, ted->ted_grant);
/* check whether we can fill the gap with unallocated
* grant */
if (*left > (oa->o_grant_used - ted->ted_grant)) {
/* ouf .. we are safe for now */
granted = ted->ted_grant;
ungranted = oa->o_grant_used - granted;
*left -= ungranted;
skip = true;
}
/* too bad, but we cannot afford to blow up our grant
* accounting. The loop below will handle each rnb in
* case by case. */
}
}
for (i = 0; i < niocount; i++) {
int bytes;
if ((rnb[i].rnb_flags & OBD_BRW_FROM_GRANT)) {
if (skip) {
rnb[i].rnb_flags |= OBD_BRW_GRANTED;
continue;
}
/* compute how much grant space is actually needed for
* this rnb, inflate grant if required */
bytes = tgt_grant_rnb_size(exp, lut, &rnb[i]);
if (ted->ted_grant >= granted + bytes) {
granted += bytes;
rnb[i].rnb_flags |= OBD_BRW_GRANTED;
continue;
}
CDEBUG(D_CACHE, "%s: cli %s/%p claims %ld+%d GRANT, real grant %lu idx %d\n",
obd->obd_name,
exp->exp_client_uuid.uuid, exp, granted, bytes,
ted->ted_grant, i);
}
if (test_bit(OBDF_RECOVERING, obd->obd_flags))
CERROR("%s: cli %s is replaying OST_WRITE while one rnb hasn't OBD_BRW_FROM_GRANT set (0x%x)\n",
obd->obd_name, exp->exp_client_uuid.uuid,
rnb[i].rnb_flags);
/* Consume grant space on the server.
* Unlike above, tgt_grant_rnb_size() is called with exp = NULL
* so that the required grant space isn't inflated. This is
* done on purpose since the server can deal with large block
* size, unlike some clients */
bytes = tgt_grant_rnb_size(NULL, lut, &rnb[i]);
if (*left > bytes) {
/* if enough space, pretend it was granted */
ungranted += bytes;
*left -= bytes;
rnb[i].rnb_flags |= OBD_BRW_GRANTED;
continue;
}
/* We can't check for already-mapped blocks here (make sense
* when backend filesystem does not use COW) as it requires
* dropping the grant lock.
* Instead, we clear OBD_BRW_GRANTED and in that case we need
* to go through and verify if all of the blocks not marked
* BRW_GRANTED are already mapped and we can ignore this error.
*/
rnb[i].rnb_flags &= ~OBD_BRW_GRANTED;
CDEBUG(D_CACHE, "%s: cli %s/%p idx %d no space for %d\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp, i, bytes);
}
/* record in o_grant_used the actual space reserved for the I/O, will be
* used later in tgt_grant_commmit() */
oa->o_grant_used = granted + ungranted;
/* record space used for the I/O, will be used in tgt_grant_commmit() */
/* Now substract what the clients has used already. We don't subtract
* this from the tot_granted yet, so that other client's can't grab
* that space before we have actually allocated our blocks. That
* happens in tgt_grant_commit() after the writes are done. */
ted->ted_grant -= granted;
ted->ted_pending += oa->o_grant_used;
tgd->tgd_tot_granted += ungranted;
tgd->tgd_tot_pending += oa->o_grant_used;
CDEBUG(D_CACHE,
"%s: cli %s/%p granted: %lu ungranted: %lu grant: %lu dirty: %lu\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp,
granted, ungranted, ted->ted_grant, ted->ted_dirty);
if (test_bit(OBDF_RECOVERING, obd->obd_flags) ||
(oa->o_valid & OBD_MD_FLGRANT) == 0)
/* don't update dirty accounting during recovery or
* if grant information got discarded (e.g. during resend) */
RETURN_EXIT;
if (ted->ted_dirty < granted) {
CWARN("%s: cli %s/%p claims granted %lu > ted_dirty %lu\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp,
granted, ted->ted_dirty);
granted = ted->ted_dirty;
}
tgd->tgd_tot_dirty -= granted;
ted->ted_dirty -= granted;
if (ted->ted_dirty < 0 || ted->ted_grant < 0 || ted->ted_pending < 0) {
CERROR("%s: cli %s/%p dirty %ld pend %ld grant %ld\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp,
ted->ted_dirty, ted->ted_pending, ted->ted_grant);
spin_unlock(&tgd->tgd_grant_lock);
LBUG();
}
EXIT;
}
/**
* Allocate additional grant space to a client
*
* Calculate how much grant space to return to client, based on how much space
* is currently free and how much of that is already granted.
* Caller must hold tgd_grant_lock spinlock.
*
* \param[in] exp export of the client which sent the request
* \param[in] curgrant current grant claimed by the client
* \param[in] want how much grant space the client would like to
* have
* \param[in] left remaining free space with granted space taken
* out
* \param[in] chunk grant allocation unit
* \param[in] conservative if set to true, the server should be cautious
* and limit how much space is granted back to the
* client. Otherwise, the server should try hard to
* satisfy the client request.
*
* \retval amount of grant space allocated
*/
static long tgt_grant_alloc(struct obd_export *exp, u64 curgrant,
u64 want, u64 left, long chunk,
bool conservative)
{
struct obd_device *obd = exp->exp_obd;
struct tg_grants_data *tgd = &obd2obt(obd)->obt_lut->lut_tgd;
struct tg_export_data *ted = &exp->exp_target_data;
u64 grant;
ENTRY;
if (CFS_FAIL_CHECK(OBD_FAIL_TGT_NO_GRANT))
RETURN(0);
/* When tgd_grant_compat_disable is set, we don't grant any space to
* clients not supporting OBD_CONNECT_GRANT_PARAM.
* Otherwise, space granted to such a client is inflated since it
* consumes PAGE_SIZE of grant space per block */
if ((obd->obd_self_export != exp && !exp_grant_param_supp(exp) &&
tgd->tgd_grant_compat_disable) || left == 0 || exp->exp_failed)
RETURN(0);
if (want > OBD_MAX_GRANT) {
CERROR("%s: client %s/%p requesting > max (%lu), %llu\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp,
OBD_MAX_GRANT, want);
RETURN(0);
}
/* Grant some fraction of the client's requested grant space so that
* they are not always waiting for write credits (not all of it to
* avoid overgranting in face of multiple RPCs in flight). This
* essentially will be able to control the OSC_MAX_RIF for a client.
*
* If we do have a large disparity between what the client thinks it
* has and what we think it has, don't grant very much and let the
* client consume its grant first. Either it just has lots of RPCs
* in flight, or it was evicted and its grants will soon be used up. */
if (curgrant >= want || curgrant >= ted->ted_grant + chunk)
RETURN(0);
if (test_bit(OBDF_RECOVERING, obd->obd_flags))
conservative = false;
if (conservative)
/* don't grant more than 1/8th of the remaining free space in
* one chunk */
left >>= 3;
grant = min(want - curgrant, left);
/* round grant up to the next block size */
grant = (grant + (1 << tgd->tgd_blockbits) - 1) &
~((1ULL << tgd->tgd_blockbits) - 1);
if (!grant)
RETURN(0);
/* Limit to grant_chunk if not reconnect/recovery */
if ((grant > chunk) && conservative)
grant = chunk;
/*
* Limit grant so that export' grant does not exceed what the
* client would like to have by more than grants for 2 full
* RPCs
*/
if (want + chunk <= ted->ted_grant)
RETURN(0);
if (ted->ted_grant + grant > want + chunk)
grant = want + chunk - ted->ted_grant;
tgd->tgd_tot_granted += grant;
ted->ted_grant += grant;
if (unlikely(ted->ted_grant < 0 || ted->ted_grant > want + chunk)) {
CERROR("%s: cli %s/%p grant %ld want %llu current %llu\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp,
ted->ted_grant, want, curgrant);
if (lbug_on_grant_miscount) {
spin_unlock(&tgd->tgd_grant_lock);
LBUG();
}
}
CDEBUG(D_CACHE,
"%s: cli %s/%p wants: %llu current grant %llu granting: %llu\n",
obd->obd_name, exp->exp_client_uuid.uuid,
exp, want, curgrant, grant);
CDEBUG(D_CACHE,
"%s: cli %s/%p tot cached:%llu granted:%llu num_exports: %d\n",
obd->obd_name, exp->exp_client_uuid.uuid,
exp, tgd->tgd_tot_dirty, tgd->tgd_tot_granted,
obd->obd_num_exports);
RETURN(grant);
}
/**
* Deallocate space granted to a client
*
* This is used when write RPC fails. Server needs to update grant
* counters as a client won't get addiitional grants.
*
* \param[in] exp export of the client which sent the request
* \param[in] granted grants allocated via tgt_grant_prepare_write
*/
void tgt_grant_dealloc(struct obd_export *exp, struct obdo *oa)
{
struct tg_grants_data *tgd = &obd2obt(exp->exp_obd)->obt_lut->lut_tgd;
struct tg_export_data *ted = &exp->exp_target_data;
int disconnected;
if (!(oa->o_valid & OBD_MD_FLGRANT))
return;
spin_lock(&tgd->tgd_grant_lock);
spin_lock(&exp->exp_lock);
disconnected = exp->exp_disconnected;
spin_unlock(&exp->exp_lock);
if (!disconnected) {
tgd->tgd_tot_granted -= oa->o_grant;
ted->ted_grant -= oa->o_grant;
oa->o_grant = 0;
}
spin_unlock(&tgd->tgd_grant_lock);
}
EXPORT_SYMBOL(tgt_grant_dealloc);
/**
* Handle grant space allocation on client connection & reconnection.
*
* A new non-readonly connection gets an initial grant allocation equals to
* tgt_grant_chunk() (i.e. twice the max BRW size in most of the cases).
* On reconnection, grant counters between client & target are resynchronized
* and additional space might be granted back if possible.
*
* \param[in] env LU environment provided by the caller
* \param[in] exp client's export which is (re)connecting
* \param[in,out] data obd_connect_data structure sent by the client in the
* connect request
* \param[in] new_conn must set to true if this is a new connection and false
* for a reconnection
*/
void tgt_grant_connect(const struct lu_env *env, struct obd_export *exp,
struct obd_connect_data *data, bool new_conn)
{
struct lu_target *lut = obd2obt(exp->exp_obd)->obt_lut;
struct tg_grants_data *tgd = &lut->lut_tgd;
struct tg_export_data *ted = &exp->exp_target_data;
u64 left = 0;
u64 want;
long chunk;
int from_cache;
int force = 0; /* can use cached data */
/* don't grant space to client with read-only access */
if (OCD_HAS_FLAG(data, RDONLY) ||
(!OCD_HAS_FLAG(data, GRANT_PARAM) &&
tgd->tgd_grant_compat_disable)) {
data->ocd_grant = 0;
data->ocd_connect_flags &= ~(OBD_CONNECT_GRANT |
OBD_CONNECT_GRANT_PARAM);
RETURN_EXIT;
}
if (OCD_HAS_FLAG(data, GRANT_PARAM))
want = data->ocd_grant;
else
want = tgt_grant_inflate(tgd, data->ocd_grant);
chunk = tgt_grant_chunk(exp, lut, data);
refresh:
tgt_grant_statfs(env, exp, force, &from_cache);
spin_lock(&tgd->tgd_grant_lock);
/* Grab free space from cached info and take out space already granted
* to clients as well as reserved space */
left = tgt_grant_space_left(exp);
/* get fresh statfs data if we are short in ungranted space */
if (from_cache && left < 32 * chunk) {
spin_unlock(&tgd->tgd_grant_lock);
CDEBUG(D_CACHE, "fs has no space left and statfs too old\n");
force = 1;
goto refresh;
}
tgt_grant_alloc(exp, (u64)ted->ted_grant, want, left, chunk, new_conn);
/* return to client its current grant */
if (OCD_HAS_FLAG(data, GRANT_PARAM))
data->ocd_grant = ted->ted_grant;
else
/* deflate grant */
data->ocd_grant = tgt_grant_deflate(tgd, (u64)ted->ted_grant);
/* reset dirty accounting */
tgd->tgd_tot_dirty -= ted->ted_dirty;
ted->ted_dirty = 0;
if (new_conn && OCD_HAS_FLAG(data, GRANT))
tgd->tgd_tot_granted_clients++;
spin_unlock(&tgd->tgd_grant_lock);
CDEBUG(D_CACHE, "%s: cli %s/%p ocd_grant: %d want: %llu left: %llu\n",
exp->exp_obd->obd_name, exp->exp_client_uuid.uuid,
exp, data->ocd_grant, want, left);
EXIT;
}
EXPORT_SYMBOL(tgt_grant_connect);
/**
* Release all grant space attached to a given export.
*
* Remove a client from the grant accounting totals. We also remove
* the export from the obd device under the osfs and dev locks to ensure
* that the tgt_grant_sanity_check() calculations are always valid.
* The client should do something similar when it invalidates its import.
*
* \param[in] exp client's export to remove from grant accounting
*/
void tgt_grant_discard(struct obd_export *exp)
{
struct obd_device *obd = exp->exp_obd;
struct lu_target *lut = class_exp2tgt(exp);
struct tg_export_data *ted = &exp->exp_target_data;
struct tg_grants_data *tgd;
if (!lut)
return;
tgd = &lut->lut_tgd;
spin_lock(&tgd->tgd_grant_lock);
if (unlikely(tgd->tgd_tot_granted < ted->ted_grant ||
tgd->tgd_tot_dirty < ted->ted_dirty)) {
struct obd_export *e;
u64 ttg = 0;
u64 ttd = 0;
list_for_each_entry(e, &obd->obd_exports, exp_obd_chain) {
LASSERT(exp != e);
ttg += e->exp_target_data.ted_grant;
ttg += e->exp_target_data.ted_pending;
ttd += e->exp_target_data.ted_dirty;
}
if (tgd->tgd_tot_granted < ted->ted_grant)
CERROR("%s: cli %s/%p: tot_granted %llu < ted_grant %ld, corrected to %llu\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp,
tgd->tgd_tot_granted, ted->ted_grant, ttg);
if (tgd->tgd_tot_dirty < ted->ted_dirty)
CERROR("%s: cli %s/%p: tot_dirty %llu < ted_dirty %ld, corrected to %llu\n",
obd->obd_name, exp->exp_client_uuid.uuid, exp,
tgd->tgd_tot_dirty, ted->ted_dirty, ttd);
tgd->tgd_tot_granted = ttg;
tgd->tgd_tot_dirty = ttd;
} else {
tgd->tgd_tot_granted -= ted->ted_grant;
tgd->tgd_tot_dirty -= ted->ted_dirty;
}
ted->ted_grant = 0;
ted->ted_dirty = 0;
if (tgd->tgd_tot_pending < ted->ted_pending) {
CERROR("%s: tot_pending %llu < cli %s/%p ted_pending %ld\n",
obd->obd_name, tgd->tgd_tot_pending,
exp->exp_client_uuid.uuid, exp, ted->ted_pending);
}
/* tgd_tot_pending is handled in tgt_grant_commit as bulk
* commmits */
spin_unlock(&tgd->tgd_grant_lock);
}
EXPORT_SYMBOL(tgt_grant_discard);
/**
* Process grant information from incoming bulk read request.
*
* Extract grant information packed in obdo structure (OBD_MD_FLGRANT set in
* o_valid). Bulk reads usually comes with grant announcements (number of dirty
* blocks, remaining amount of grant space, ...) and could also include a grant
* shrink request. Unlike bulk write, no additional grant space is returned on
* bulk read request.
*
* \param[in] env is the lu environment provided by the caller
* \param[in] exp is the export of the client which sent the request
* \param[in,out] oa is the incoming obdo sent by the client
*/
void tgt_grant_prepare_read(const struct lu_env *env,
struct obd_export *exp, struct obdo *oa)
{
struct lu_target *lut = obd2obt(exp->exp_obd)->obt_lut;
struct tg_grants_data *tgd = &lut->lut_tgd;
int do_shrink;
u64 left = 0;
ENTRY;
if (!oa)
RETURN_EXIT;
if ((oa->o_valid & OBD_MD_FLGRANT) == 0)
/* The read request does not contain any grant
* information */
RETURN_EXIT;
if ((oa->o_valid & OBD_MD_FLFLAGS) &&
(oa->o_flags & OBD_FL_SHRINK_GRANT)) {
/* To process grant shrink request, we need to know how much
* available space remains on the backend filesystem.
* Shrink requests are not so common, we always get fresh
* statfs information. */
tgt_grant_statfs(env, exp, 1, NULL);
/* protect all grant counters */
spin_lock(&tgd->tgd_grant_lock);
/* Grab free space from cached statfs data and take out space
* already granted to clients as well as reserved space */
left = tgt_grant_space_left(exp);
/* all set now to proceed with shrinking */
do_shrink = 1;
} else {
/* no grant shrinking request packed in the obdo and
* since we don't grant space back on reads, no point
* in running statfs, so just skip it and process
* incoming grant data directly. */
spin_lock(&tgd->tgd_grant_lock);
do_shrink = 0;
}
/* extract incoming grant information provided by the client and
* inflate grant counters if required */
tgt_grant_incoming(env, exp, oa, tgt_grant_chunk(exp, lut, NULL));
/* unlike writes, we don't return grants back on reads unless a grant
* shrink request was packed and we decided to turn it down. */
if (do_shrink)
tgt_grant_shrink(exp, oa, left);
else
oa->o_grant = 0;
if (!exp_grant_param_supp(exp))
oa->o_grant = tgt_grant_deflate(tgd, oa->o_grant);
spin_unlock(&tgd->tgd_grant_lock);
EXIT;
}
EXPORT_SYMBOL(tgt_grant_prepare_read);
/**
* Process grant information from incoming bulk write request.
*
* This function extracts client's grant announcements from incoming bulk write
* request and attempts to allocate grant space for network buffers that need it
* (i.e. OBD_BRW_FROM_GRANT not set in rnb_fags).
* Network buffers which aren't granted the OBD_BRW_GRANTED flag should not
* proceed further and should fail with -ENOSPC.
* Whenever possible, additional grant space will be returned to the client
* in the bulk write reply.
* tgt_grant_prepare_write() must be called before writting any buffers to
* the backend storage. This function works in pair with tgt_grant_commit()
* which must be invoked once all buffers have been written to disk in order
* to release space from the pending grant counter.
*
* \param[in] env LU environment provided by the caller
* \param[in] exp export of the client which sent the request
* \param[in] oa incoming obdo sent by the client
* \param[in] rnb list of network buffers
* \param[in] niocount number of network buffers in the list
*/
void tgt_grant_prepare_write(const struct lu_env *env,
struct obd_export *exp, struct obdo *oa,
struct niobuf_remote *rnb, int niocount)
{
struct obd_device *obd = exp->exp_obd;
struct lu_target *lut = obd2obt(obd)->obt_lut;
struct tg_grants_data *tgd = &lut->lut_tgd;
u64 left;
int from_cache;
int force = 0; /* can use cached data intially */
long chunk = tgt_grant_chunk(exp, lut, NULL);
ENTRY;
refresh:
/* get statfs information from OSD layer */
tgt_grant_statfs(env, exp, force, &from_cache);
spin_lock(&tgd->tgd_grant_lock); /* protect all grant counters */
/* Grab free space from cached statfs data and take out space already
* granted to clients as well as reserved space */
left = tgt_grant_space_left(exp);
/* Get fresh statfs data if we are short in ungranted space */
if (from_cache && left < 32 * chunk) {
spin_unlock(&tgd->tgd_grant_lock);
CDEBUG(D_CACHE, "%s: fs has no space left and statfs too old\n",
obd->obd_name);
force = 1;
goto refresh;
}
/* When close to free space exhaustion, trigger a sync to force
* writeback cache to consume required space immediately and release as
* much space as possible. */
if (!test_bit(OBDF_RECOVERING, obd->obd_flags) && force != 2 && left < chunk) {
bool from_grant = true;
int i;
/* That said, it is worth running a sync only if some pages did
* not consume grant space on the client and could thus fail
* with ENOSPC later in tgt_grant_check() */
for (i = 0; i < niocount; i++)
if (!(rnb[i].rnb_flags & OBD_BRW_FROM_GRANT))
from_grant = false;
if (!from_grant) {
/* at least one network buffer requires acquiring grant
* space on the server */
spin_unlock(&tgd->tgd_grant_lock);
/* discard errors, at least we tried ... */
dt_sync(env, lut->lut_bottom);
force = 2;
goto refresh;
}
}
/* extract incoming grant information provided by the client,
* and inflate grant counters if required */
tgt_grant_incoming(env, exp, oa, chunk);
/* check limit */
tgt_grant_check(env, exp, oa, rnb, niocount, &left);
if (!(oa->o_valid & OBD_MD_FLGRANT)) {
spin_unlock(&tgd->tgd_grant_lock);
RETURN_EXIT;
}
/* if OBD_FL_SHRINK_GRANT is set, the client is willing to release some
* grant space. */
if ((oa->o_valid & OBD_MD_FLFLAGS) &&
(oa->o_flags & OBD_FL_SHRINK_GRANT))
tgt_grant_shrink(exp, oa, left);
else
/* grant more space back to the client if possible */
oa->o_grant = tgt_grant_alloc(exp, oa->o_grant, oa->o_undirty,
left, chunk, true);
if (!exp_grant_param_supp(exp))
oa->o_grant = tgt_grant_deflate(tgd, oa->o_grant);
spin_unlock(&tgd->tgd_grant_lock);
EXIT;
}
EXPORT_SYMBOL(tgt_grant_prepare_write);
/**
* Consume grant space reserved for object creation.
*
* Grant space is allocated to the local self export for object precreation.
* This is required to prevent object precreation from consuming grant space
* allocated to client nodes for the data writeback cache.
* This function consumes enough space to create \a nr objects and allocates
* more grant space to the self export for future precreation requests, if
* possible.
*
* \param[in] env LU environment provided by the caller
* \param[in] exp export holding the grant space for precreation (= self
* export currently)
* \param[in] nr number of objects to be created
*
* \retval >= 0 amount of grant space allocated to the precreate request
* \retval -ENOSPC on failure
*/
long tgt_grant_create(const struct lu_env *env, struct obd_export *exp, s64 *nr)
{
struct lu_target *lut = obd2obt(exp->exp_obd)->obt_lut;
struct tg_grants_data *tgd = &lut->lut_tgd;
struct tg_export_data *ted = &exp->exp_target_data;
u64 left = 0;
unsigned long wanted;
unsigned long granted;
ENTRY;
if (test_bit(OBDF_RECOVERING, exp->exp_obd->obd_flags) ||
lut->lut_dt_conf.ddp_inodespace == 0)
/* don't enforce grant during recovery */
RETURN(0);
/* Update statfs data if required */
tgt_grant_statfs(env, exp, 1, NULL);
/* protect all grant counters */
spin_lock(&tgd->tgd_grant_lock);
/* fail precreate request if there is not enough blocks available for
* writing */
if (tgd->tgd_osfs.os_bavail - (ted->ted_grant >> tgd->tgd_blockbits) <
(tgd->tgd_osfs.os_blocks >> 10)) {
spin_unlock(&tgd->tgd_grant_lock);
CDEBUG(D_RPCTRACE, "%s: not enough space for create %llu\n",
exp->exp_obd->obd_name,
tgd->tgd_osfs.os_bavail * tgd->tgd_osfs.os_blocks);
RETURN(-ENOSPC);
}
/* Grab free space from cached statfs data and take out space
* already granted to clients as well as reserved space */
left = tgt_grant_space_left(exp);
/* compute how much space is required to handle the precreation
* request */
wanted = *nr * lut->lut_dt_conf.ddp_inodespace;
if (wanted > ted->ted_grant + left) {
/* that's beyond what remains, adjust the number of objects that
* can be safely precreated */
wanted = ted->ted_grant + left;
*nr = wanted / lut->lut_dt_conf.ddp_inodespace;
if (*nr == 0) {
/* we really have no space any more for precreation,
* fail the precreate request with ENOSPC */
spin_unlock(&tgd->tgd_grant_lock);
RETURN(-ENOSPC);
}
/* compute space needed for the new number of creations */
wanted = *nr * lut->lut_dt_conf.ddp_inodespace;
}
LASSERT(wanted <= ted->ted_grant + left);
if (wanted <= ted->ted_grant) {
/* we've enough grant space to handle this precreate request */
ted->ted_grant -= wanted;
} else {
/* we need to take some space from the ungranted pool */
tgd->tgd_tot_granted += wanted - ted->ted_grant;
left -= wanted - ted->ted_grant;
ted->ted_grant = 0;
}
granted = wanted;
ted->ted_pending += granted;
tgd->tgd_tot_pending += granted;
/* grant more space for precreate purpose if possible. */
wanted = OST_MAX_PRECREATE * lut->lut_dt_conf.ddp_inodespace / 2;
if (wanted > ted->ted_grant) {
long chunk;
/* always try to book enough space to handle a large precreate
* request */
chunk = tgt_grant_chunk(exp, lut, NULL);
wanted -= ted->ted_grant;
tgt_grant_alloc(exp, ted->ted_grant, wanted, left, chunk,
false);
}
spin_unlock(&tgd->tgd_grant_lock);
RETURN(granted);
}
EXPORT_SYMBOL(tgt_grant_create);
/**
* Release grant space added to the pending counter by tgt_grant_prepare_write()
*
* Update pending grant counter once buffers have been written to the disk.
*
* \param[in] exp export of the client which sent the request
* \param[in] pending amount of reserved space to be released
* \param[in] rc return code of pre-commit operations
*/
void tgt_grant_commit(struct obd_export *exp, unsigned long pending,
int rc)
{
struct tg_grants_data *tgd = &obd2obt(exp->exp_obd)->obt_lut->lut_tgd;
ENTRY;
/* get space accounted in tot_pending for the I/O, set in
* tgt_grant_check() */
if (pending == 0)
RETURN_EXIT;
spin_lock(&tgd->tgd_grant_lock);
/* Don't update statfs data for errors raised before commit (e.g.
* bulk transfer failed, ...) since we know those writes have not been
* processed. For other errors hit during commit, we cannot really tell
* whether or not something was written, so we update statfs data.
* In any case, this should not be fatal since we always get fresh
* statfs data before failing a request with ENOSPC */
if (rc == 0) {
spin_lock(&tgd->tgd_osfs_lock);
/* Take pending out of cached statfs data */
tgd->tgd_osfs.os_bavail -= min_t(u64,
tgd->tgd_osfs.os_bavail,
pending >> tgd->tgd_blockbits);
if (tgd->tgd_statfs_inflight)
/* someone is running statfs and want to be notified of
* writes happening meanwhile */
tgd->tgd_osfs_inflight += pending;
spin_unlock(&tgd->tgd_osfs_lock);
}
if (exp->exp_target_data.ted_pending < pending) {
CERROR("%s: cli %s/%p ted_pending(%lu) < grant_used(%lu)\n",
exp->exp_obd->obd_name, exp->exp_client_uuid.uuid, exp,
exp->exp_target_data.ted_pending, pending);
spin_unlock(&tgd->tgd_grant_lock);
LBUG();
}
exp->exp_target_data.ted_pending -= pending;
if (tgd->tgd_tot_granted < pending) {
CERROR("%s: cli %s/%p tot_granted(%llu) < grant_used(%lu)\n",
exp->exp_obd->obd_name, exp->exp_client_uuid.uuid, exp,
tgd->tgd_tot_granted, pending);
spin_unlock(&tgd->tgd_grant_lock);
LBUG();
}
tgd->tgd_tot_granted -= pending;
if (tgd->tgd_tot_pending < pending) {
CERROR("%s: cli %s/%p tot_pending(%llu) < grant_used(%lu)\n",
exp->exp_obd->obd_name, exp->exp_client_uuid.uuid, exp,
tgd->tgd_tot_pending, pending);
spin_unlock(&tgd->tgd_grant_lock);
LBUG();
}
tgd->tgd_tot_pending -= pending;
spin_unlock(&tgd->tgd_grant_lock);
EXIT;
}
EXPORT_SYMBOL(tgt_grant_commit);
struct tgt_grant_cb {
/* commit callback structure */
struct dt_txn_commit_cb tgc_cb;
/* export associated with the bulk write */
struct obd_export *tgc_exp;
/* pending grant to be released */
unsigned long tgc_granted;
};
/**
* Callback function for grant releasing
*
* Release grant space reserved by the client node.
*
* \param[in] env execution environment
* \param[in] th transaction handle
* \param[in] cb callback data
* \param[in] err error code
*/
static void tgt_grant_commit_cb(struct lu_env *env, struct thandle *th,
struct dt_txn_commit_cb *cb, int err)
{
struct tgt_grant_cb *tgc;
tgc = container_of(cb, struct tgt_grant_cb, tgc_cb);
tgt_grant_commit(tgc->tgc_exp, tgc->tgc_granted, err);
class_export_cb_put(tgc->tgc_exp);
OBD_FREE_PTR(tgc);
}
/**
* Add callback for grant releasing
*
* Register a commit callback to release grant space.
*
* \param[in] th transaction handle
* \param[in] exp OBD export of client
* \param[in] granted amount of grant space to be released upon commit
*
* \retval 0 on successful callback adding
* \retval negative value on error
*/
int tgt_grant_commit_cb_add(struct thandle *th, struct obd_export *exp,
unsigned long granted)
{
struct tgt_grant_cb *tgc;
struct dt_txn_commit_cb *dcb;
int rc;
ENTRY;
OBD_ALLOC_PTR(tgc);
if (tgc == NULL)
RETURN(-ENOMEM);
tgc->tgc_exp = class_export_cb_get(exp);
tgc->tgc_granted = granted;
dcb = &tgc->tgc_cb;
dcb->dcb_func = tgt_grant_commit_cb;
INIT_LIST_HEAD(&dcb->dcb_linkage);
strscpy(dcb->dcb_name, "tgt_grant_commit_cb", sizeof(dcb->dcb_name));
rc = dt_trans_cb_add(th, dcb);
if (rc) {
class_export_cb_put(tgc->tgc_exp);
OBD_FREE_PTR(tgc);
}
RETURN(rc);
}
EXPORT_SYMBOL(tgt_grant_commit_cb_add);
/**
* Show estimate of total amount of dirty data on clients.
*
* @kobj kobject embedded in obd_device
* @attr unused
* @buf buf used by sysfs to print out data
*
* Return: 0 on success
* negative value on error
*/
ssize_t tot_dirty_show(struct kobject *kobj, struct attribute *attr,
char *buf)
{
struct obd_device *obd = container_of(kobj, struct obd_device,
obd_kset.kobj);
struct tg_grants_data *tgd;
tgd = &obd2obt(obd)->obt_lut->lut_tgd;
return scnprintf(buf, PAGE_SIZE, "%llu\n", tgd->tgd_tot_dirty);
}
EXPORT_SYMBOL(tot_dirty_show);
/**
* Show total amount of space granted to clients.
*
* @kobj kobject embedded in obd_device
* @attr unused
* @buf buf used by sysfs to print out data
*
* Return: 0 on success
* negative value on error
*/
ssize_t tot_granted_show(struct kobject *kobj, struct attribute *attr,
char *buf)
{
struct obd_device *obd = container_of(kobj, struct obd_device,
obd_kset.kobj);
struct tg_grants_data *tgd;
tgd = &obd2obt(obd)->obt_lut->lut_tgd;
return scnprintf(buf, PAGE_SIZE, "%llu\n", tgd->tgd_tot_granted);
}
EXPORT_SYMBOL(tot_granted_show);
/**
* Show total amount of space used by IO in progress.
*
* @kobj kobject embedded in obd_device
* @attr unused
* @buf buf used by sysfs to print out data
*
* Return: 0 on success
* negative value on error
*/
ssize_t tot_pending_show(struct kobject *kobj, struct attribute *attr,
char *buf)
{
struct obd_device *obd = container_of(kobj, struct obd_device,
obd_kset.kobj);
struct tg_grants_data *tgd;
tgd = &obd2obt(obd)->obt_lut->lut_tgd;
return scnprintf(buf, PAGE_SIZE, "%llu\n", tgd->tgd_tot_pending);
}
EXPORT_SYMBOL(tot_pending_show);
/**
* Show if grants compatibility mode is disabled.
*
* When tgd_grant_compat_disable is set, we don't grant any space to clients
* not supporting OBD_CONNECT_GRANT_PARAM. Otherwise, space granted to such
* a client is inflated since it consumes PAGE_SIZE of grant space per
* block, (i.e. typically 4kB units), but underlaying file system might have
* block size bigger than page size, e.g. ZFS. See LU-2049 for details.
*
* @kobj kobject embedded in obd_device
* @attr unused
* @buf buf used by sysfs to print out data
*
* Return: string length of @buf output on success
*/
ssize_t grant_compat_disable_show(struct kobject *kobj, struct attribute *attr,
char *buf)
{
struct obd_device *obd = container_of(kobj, struct obd_device,
obd_kset.kobj);
struct tg_grants_data *tgd = &obd2obt(obd)->obt_lut->lut_tgd;
return scnprintf(buf, PAGE_SIZE, "%u\n", tgd->tgd_grant_compat_disable);
}
EXPORT_SYMBOL(grant_compat_disable_show);
/**
* Change grant compatibility mode.
*
* Setting tgd_grant_compat_disable prohibit any space granting to clients
* not supporting OBD_CONNECT_GRANT_PARAM. See details above.
*
* @kobj kobject embedded in obd_device
* @attr unused
* @buffer string which represents mode
* 1: disable compatibility mode
* 0: enable compatibility mode
* @count @buffer length
*
* Return: @count on success
* negative number on error
*/
ssize_t grant_compat_disable_store(struct kobject *kobj,
struct attribute *attr,
const char *buffer, size_t count)
{
struct obd_device *obd = container_of(kobj, struct obd_device,
obd_kset.kobj);
struct tg_grants_data *tgd = &obd2obt(obd)->obt_lut->lut_tgd;
bool val;
int rc;
rc = kstrtobool(buffer, &val);
if (rc)
return rc;
tgd->tgd_grant_compat_disable = val;
return count;
}
EXPORT_SYMBOL(grant_compat_disable_store);
