XrdThrottleManager Class Reference

#include <XrdThrottleManager.hh>

Collaboration diagram for XrdThrottleManager:

Public Member Functions
	XrdThrottleManager (XrdSysError *lP, XrdOucTrace *tP)
	~XrdThrottleManager ()
void	Apply (int reqsize, int reqops, int uid)
bool	CheckLoadShed (const std::string &opaque)
bool	CloseFile (const std::string &entity)
void	FromConfig (XrdThrottle::Configuration &config)
std::tuple< std::string, uint16_t >	GetUserInfo (const XrdSecEntity *client)
void	Init ()
bool	IsThrottling ()
bool	OpenFile (const std::string &entity, std::string &open_error_message)
void	PerformLoadShed (const std::string &opaque, std::string &host, unsigned &port)
void	PrepLoadShed (const char *opaque, std::string &lsOpaque)
void	SetLoadShed (std::string &hostname, unsigned port, unsigned frequency)
void	SetMaxConns (unsigned long max_conns)
void	SetMaxOpen (unsigned long max_open)
void	SetMaxWait (unsigned long max_wait)
void	SetMonitor (XrdXrootdGStream *gstream)
void	SetThrottles (float reqbyterate, float reqoprate, int concurrency, float interval_length)
XrdThrottleTimer	StartIOTimer (uint16_t uid, bool &ok)

Protected Member Functions
void	StopIOTimer (std::chrono::steady_clock::duration &event_duration, uint16_t uid)

Friends
class	XrdThrottleTimer

Detailed Description

Definition at line 53 of file XrdThrottleManager.hh.

Constructor & Destructor Documentation

XrdThrottleManager()

XrdThrottleManager::XrdThrottleManager	(	XrdSysError *	lP,
		XrdOucTrace *	tP
	)

Definition at line 44 of file XrdThrottleManager.cc.

                                                                       :
   m_trace(tP),
   m_log(lP),
   m_interval_length_seconds(1.0),
   m_bytes_per_second(-1),
   m_ops_per_second(-1),
   m_concurrency_limit(-1),
   m_last_round_allocation(100*1024),
   m_loadshed_host(""),
   m_loadshed_port(0),
   m_loadshed_frequency(0)
{
   // Until the first recompute fills them in, no user is in the wake order.
   std::fill(m_wake_order_0.begin(), m_wake_order_0.end(), -1);
   std::fill(m_wake_order_1.begin(), m_wake_order_1.end(), -1);
}

~XrdThrottleManager()

XrdThrottleManager::~XrdThrottleManager ( )

inline

Definition at line 107 of file XrdThrottleManager.hh.

{} // The buffmanager is never deleted

Member Function Documentation

Apply()

void XrdThrottleManager::Apply	(	int	reqsize,
		int	reqops,
		int	uid
	)

Definition at line 325 of file XrdThrottleManager.cc.

{
   if (m_bytes_per_second < 0)
      reqsize = 0;
   if (m_ops_per_second < 0)
      reqops = 0;
   while (reqsize || reqops)
   {
      // Subtract the requested out of the shares
      AtomicBeg(m_compute_var);
      GetShares(m_primary_bytes_shares[uid], reqsize);
      if (reqsize)
      {
         TRACE(BANDWIDTH, "Using secondary shares; request has " << reqsize << " bytes left.");
         GetShares(m_secondary_bytes_shares[uid], reqsize);
         TRACE(BANDWIDTH, "Finished with secondary shares; request has " << reqsize << " bytes left.");
      }
      else
      {
         TRACE(BANDWIDTH, "Filled byte shares out of primary; " << m_primary_bytes_shares[uid] << " left.");
      }
      GetShares(m_primary_ops_shares[uid], reqops);
      if (reqops)
      {
         GetShares(m_secondary_ops_shares[uid], reqops);
      }
      StealShares(uid, reqsize, reqops);
      AtomicEnd(m_compute_var);
 
      if (reqsize || reqops)
      {
         if (reqsize) TRACE(BANDWIDTH, "Sleeping to wait for throttle fairshare.");
         if (reqops) TRACE(IOPS, "Sleeping to wait for throttle fairshare.");
         m_compute_var.Wait();
         m_loadshed_limit_hit++;
      }
   }
 
}

References AtomicBeg, AtomicEnd, TRACE, and XrdSysCondVar::Wait().

Here is the call graph for this function:

CheckLoadShed()

bool XrdThrottleManager::CheckLoadShed

(

const std::string &

opaque

)

Definition at line 777 of file XrdThrottleManager.cc.

{
   if (m_loadshed_port == 0)
   {
      return false;
   }
   if (m_loadshed_limit_hit == 0)
   {
      return false;
   }
   if (static_cast<unsigned>(rand()) % 100 > m_loadshed_frequency)
   {
      return false;
   }
   if (opaque.empty())
   {
      return false;
   }
   return true;
}

CloseFile()

bool XrdThrottleManager::CloseFile

(

const std::string &

entity

)

Definition at line 259 of file XrdThrottleManager.cc.

{
    if (m_max_open == 0 && m_max_conns == 0) return true;
 
    bool result = true;
    const std::lock_guard<std::mutex> lock(m_file_mutex);
    if (m_max_open) {
        auto iter = m_file_counters.find(entity);
        if (iter == m_file_counters.end()) {
            TRACE(FILES, "WARNING: User " << entity << " closed a file but throttle plugin never saw an open file");
            result = false;
        } else if (iter->second == 0) {
            TRACE(FILES, "WARNING: User " << entity << " closed a file but throttle plugin thinks all files were already closed");
            result = false;
        } else {
            iter->second--;
        }
        if (result) TRACE(FILES, "User " << entity << " closed a file; " << iter->second <<
                                 " remain open");
    }
 
    if (m_max_conns) {
        auto pid = XrdSysThread::Num();
        auto conn_iter = m_active_conns.find(entity);
        auto conn_count_iter = m_conn_counters.find(entity);
        if (conn_iter == m_active_conns.end() || !(conn_iter->second)) {
            TRACE(CONNS, "WARNING: User " << entity << " closed a file on a connection we are not"
                " tracking");
            return false;
        }
        auto pid_iter = conn_iter->second->find(pid);
        if (pid_iter == conn_iter->second->end()) {
            TRACE(CONNS, "WARNING: User " << entity << " closed a file on a connection we are not"
                " tracking");
            return false;
        }
        if (pid_iter->second == 0) {
            TRACE(CONNS, "WARNING: User " << entity << " closed a file on connection the throttle"
                " plugin thinks was idle");
        } else {
            pid_iter->second--;
        }
        if (conn_count_iter == m_conn_counters.end()) {
            TRACE(CONNS, "WARNING: User " << entity << " closed a file but the throttle plugin never"
                " observed an open file");
        } else if (pid_iter->second == 0) {
            if (conn_count_iter->second == 0) {
                TRACE(CONNS, "WARNING: User " << entity << " had a connection go idle but the "
                    " throttle plugin already thought all connections were idle");
            } else {
                conn_count_iter->second--;
                TRACE(CONNS, "User " << entity << " had connection on thread " << pid << " go idle; "
                    << conn_count_iter->second << " active connections remain");
            }
        }
    }
 
    return result;
}

References XrdSysThread::Num(), and TRACE.

Referenced by XrdThrottle::File::close(), and XrdThrottle::File::open().

Here is the call graph for this function:

Here is the caller graph for this function:

FromConfig()

void XrdThrottleManager::FromConfig

(

XrdThrottle::Configuration &

config

)

Definition at line 62 of file XrdThrottleManager.cc.

{
 
    auto max_open = config.GetMaxOpen();
    if (max_open != -1) SetMaxOpen(max_open);
    auto max_conn = config.GetMaxConn();
    if (max_conn != -1) SetMaxConns(max_conn);
    auto max_wait = config.GetMaxWait();
    if (max_wait != -1) SetMaxWait(max_wait);
 
    SetThrottles(config.GetThrottleDataRate(),
       config.GetThrottleIOPSRate(),
       config.GetThrottleConcurrency(),
       static_cast<float>(config.GetThrottleRecomputeIntervalMS())/1000.0);
 
    m_trace->What = config.GetTraceLevels();
 
    auto loadshed_host = config.GetLoadshedHost();
    auto loadshed_port = config.GetLoadshedPort();
    auto loadshed_freq = config.GetLoadshedFreq();
    if (!loadshed_host.empty() && loadshed_port > 0 && loadshed_freq > 0)
    {
       // Loadshed specified, so set it.
       SetLoadShed(loadshed_host, loadshed_port, loadshed_freq);
    }
}

References XrdThrottle::Configuration::GetLoadshedFreq(), XrdThrottle::Configuration::GetLoadshedHost(), XrdThrottle::Configuration::GetLoadshedPort(), XrdThrottle::Configuration::GetMaxConn(), XrdThrottle::Configuration::GetMaxOpen(), XrdThrottle::Configuration::GetMaxWait(), XrdThrottle::Configuration::GetThrottleConcurrency(), XrdThrottle::Configuration::GetThrottleDataRate(), XrdThrottle::Configuration::GetThrottleIOPSRate(), XrdThrottle::Configuration::GetThrottleRecomputeIntervalMS(), XrdThrottle::Configuration::GetTraceLevels(), SetLoadShed(), SetMaxConns(), SetMaxOpen(), SetMaxWait(), SetThrottles(), and XrdOucTrace::What.

Here is the call graph for this function:

GetUserInfo()

std::tuple< std::string, uint16_t > XrdThrottleManager::GetUserInfo

(

const XrdSecEntity *

client

)

Definition at line 119 of file XrdThrottleManager.cc.

                                                          {
    // client can be null, if so, return nobody
    if (!client) {
        return std::make_tuple("nobody", GetUid("nobody"));
    }
 
    // Try various potential "names" associated with the request, from the most
    // specific to most generic.
    std::string user;
 
    if (client->eaAPI && client->eaAPI->Get("token.subject", user)) {
        if (client->vorg) user = std::string(client->vorg) + ":" + user;
    } else if (client->eaAPI) {
        std::string request_name;
        if (client->eaAPI->Get("request.name", request_name) && !request_name.empty()) user = request_name;
    }
    if (user.empty()) {user = client->name ? client->name : "nobody";}
    uint16_t uid = GetUid(user.c_str());
    return std::make_tuple(user, uid);
}

References XrdSecEntity::eaAPI, XrdSecEntityAttr::Get(), XrdSecEntity::name, and XrdSecEntity::vorg.

Referenced by XrdThrottle::File::open().

Here is the call graph for this function:

Here is the caller graph for this function:

Init()

void XrdThrottleManager::Init

(

)

Definition at line 90 of file XrdThrottleManager.cc.

{
   TRACE(DEBUG, "Initializing the throttle manager.");
   // Initialize all our shares to zero.
   m_primary_bytes_shares.resize(m_max_users);
   m_secondary_bytes_shares.resize(m_max_users);
   m_primary_ops_shares.resize(m_max_users);
   m_secondary_ops_shares.resize(m_max_users);
   for (auto & waiter : m_waiter_info) {
      waiter.m_manager = this;
   }
 
   // Allocate each user 100KB and 10 ops to bootstrap;
   for (int i=0; i<m_max_users; i++)
   {
      m_primary_bytes_shares[i] = m_last_round_allocation;
      m_secondary_bytes_shares[i] = 0;
      m_primary_ops_shares[i] = 10;
      m_secondary_ops_shares[i] = 0;
   }
 
   int rc;
   pthread_t tid;
   if ((rc = XrdSysThread::Run(&tid, XrdThrottleManager::RecomputeBootstrap, static_cast<void *>(this), 0, "Buffer Manager throttle")))
      m_log->Emsg("ThrottleManager", rc, "create throttle thread");
 
}

References DEBUG, XrdSysError::Emsg(), XrdSysThread::Run(), and TRACE.

Here is the call graph for this function:

IsThrottling()

bool XrdThrottleManager::IsThrottling ( )

inline

Definition at line 69 of file XrdThrottleManager.hh.

{return (m_ops_per_second > 0) || (m_bytes_per_second > 0);}

OpenFile()

bool XrdThrottleManager::OpenFile	(	const std::string &	entity,
		std::string &	open_error_message
	)

Definition at line 182 of file XrdThrottleManager.cc.

{
    if (m_max_open == 0 && m_max_conns == 0) return true;
 
    const std::lock_guard<std::mutex> lock(m_file_mutex);
    auto iter = m_file_counters.find(entity);
    unsigned long cur_open_files = 0, cur_open_conns;
    if (m_max_open) {
        if (iter == m_file_counters.end()) {
            m_file_counters[entity] = 1;
            TRACE(FILES, "User " << entity << " has opened their first file");
            cur_open_files = 1;
        } else if (iter->second < m_max_open) {
            iter->second++;
            cur_open_files = iter->second;
        } else {
            std::stringstream ss;
            ss <<  "User " << entity << " has hit the limit of " << m_max_open << " open files";
            TRACE(FILES, ss.str());
            error_message = ss.str();
            return false;
        }
    }
 
    if (m_max_conns) {
        auto pid = XrdSysThread::Num();
        auto conn_iter = m_active_conns.find(entity);
        auto conn_count_iter = m_conn_counters.find(entity);
        if ((conn_count_iter != m_conn_counters.end()) && (conn_count_iter->second == m_max_conns) &&
            (conn_iter == m_active_conns.end() || ((*(conn_iter->second))[pid] == 0)))
        {
            // note: we are rolling back the increment in open files
            if (m_max_open) iter->second--;
            std::stringstream ss;
            ss << "User " << entity << " has hit the limit of " << m_max_conns <<
                " open connections";
            TRACE(CONNS, ss.str());
            error_message = ss.str();
            return false;
        }
        if (conn_iter == m_active_conns.end()) {
            std::unique_ptr<std::unordered_map<pid_t, unsigned long>> conn_map(
                new std::unordered_map<pid_t, unsigned long>());
            (*conn_map)[pid] = 1;
            m_active_conns[entity] = std::move(conn_map);
            if (conn_count_iter == m_conn_counters.end()) {
                m_conn_counters[entity] = 1;
                cur_open_conns = 1;
            } else {
                m_conn_counters[entity] ++;
                cur_open_conns = m_conn_counters[entity];
            }
        } else {
            auto pid_iter = conn_iter->second->find(pid);
            if (pid_iter == conn_iter->second->end() || pid_iter->second == 0) {
                (*(conn_iter->second))[pid] = 1;
                conn_count_iter->second++;
                cur_open_conns = conn_count_iter->second;
            } else {
                (*(conn_iter->second))[pid] ++;
                cur_open_conns = conn_count_iter->second;
           }
        }
        TRACE(CONNS, "User " << entity << " has " << cur_open_conns << " open connections");
    }
    if (m_max_open) TRACE(FILES, "User " << entity << " has " << cur_open_files << " open files");
    return true;
}

References XrdSysThread::Num(), and TRACE.

Referenced by XrdThrottle::File::open().

Here is the call graph for this function:

Here is the caller graph for this function:

PerformLoadShed()

void XrdThrottleManager::PerformLoadShed	(	const std::string &	opaque,
		std::string &	host,
		unsigned &	port
	)

Definition at line 823 of file XrdThrottleManager.cc.

{
   host = m_loadshed_host;
   host += "?";
   host += opaque;
   port = m_loadshed_port;
}

PrepLoadShed()

void XrdThrottleManager::PrepLoadShed	(	const char *	opaque,
		std::string &	lsOpaque
	)

Definition at line 799 of file XrdThrottleManager.cc.

{
   if (m_loadshed_port == 0)
   {
      return;
   }
   if (opaque && opaque[0])
   {
      XrdOucEnv env(opaque);
      // Do not load shed client if it has already been done once.
      if (env.Get("throttle.shed") != 0)
      {
         return;
      }
      lsOpaque = opaque;
      lsOpaque += "&throttle.shed=1";
   }
   else
   {
      lsOpaque = "throttle.shed=1";
   }
}

References XrdOucEnv::Get().

Referenced by XrdThrottle::File::open().

Here is the call graph for this function:

Here is the caller graph for this function:

SetLoadShed()

void XrdThrottleManager::SetLoadShed ( std::string &  hostname, unsigned  port, unsigned  frequency  )

inline

Definition at line 80 of file XrdThrottleManager.hh.

            {m_loadshed_host = hostname; m_loadshed_port = port; m_loadshed_frequency = frequency;}

Referenced by FromConfig().

Here is the caller graph for this function:

SetMaxConns()

void XrdThrottleManager::SetMaxConns ( unsigned long  max_conns )

inline

Definition at line 85 of file XrdThrottleManager.hh.

{m_max_conns = max_conns;}

Referenced by FromConfig().

Here is the caller graph for this function:

SetMaxOpen()

void XrdThrottleManager::SetMaxOpen ( unsigned long  max_open )

inline

Definition at line 83 of file XrdThrottleManager.hh.

{m_max_open = max_open;}

Referenced by FromConfig().

Here is the caller graph for this function:

SetMaxWait()

void XrdThrottleManager::SetMaxWait ( unsigned long  max_wait )

inline

Definition at line 87 of file XrdThrottleManager.hh.

{m_max_wait_time = std::chrono::seconds(max_wait);}

Referenced by FromConfig().

Here is the caller graph for this function:

SetMonitor()

void XrdThrottleManager::SetMonitor ( XrdXrootdGStream *  gstream )

inline

Definition at line 89 of file XrdThrottleManager.hh.

{m_gstream = gstream;}

SetThrottles()

void XrdThrottleManager::SetThrottles ( float  reqbyterate, float  reqoprate, int  concurrency, float  interval_length  )

inline

Definition at line 76 of file XrdThrottleManager.hh.

            {m_interval_length_seconds = interval_length; m_bytes_per_second = reqbyterate;
             m_ops_per_second = reqoprate; m_concurrency_limit = concurrency;}

Referenced by FromConfig().

Here is the caller graph for this function:

StartIOTimer()

XrdThrottleTimer XrdThrottleManager::StartIOTimer	(	uint16_t	uid,
		bool &	ok
	)

Definition at line 711 of file XrdThrottleManager.cc.

{
   int cur_counter = m_io_active.fetch_add(1, std::memory_order_acq_rel);
   m_io_total++;
 
   while (m_concurrency_limit >= 0 && cur_counter >= m_concurrency_limit)
   {
      // If the user has essentially no concurrency, then we let them
      // temporarily exceed the limit.  This prevents potential waits for
      // every single read for an infrequent user.
      if (m_waiter_info[uid].m_concurrency < 1)
      {
         break;
      }
      m_loadshed_limit_hit++;
      m_io_active.fetch_sub(1, std::memory_order_acq_rel);
      TRACE(DEBUG, "ThrottleManager (user=" << uid << "): IO concurrency limit hit; waiting for other IOs to finish.");
      ok = m_waiter_info[uid].Wait();
      if (!ok) {
        TRACE(DEBUG, "ThrottleManager (user=" << uid << "): timed out waiting for other IOs to finish.");
        return XrdThrottleTimer();
      }
      cur_counter = m_io_active.fetch_add(1, std::memory_order_acq_rel);
   }
 
   ok = true;
   return XrdThrottleTimer(this, uid);
}

References DEBUG, TRACE, and XrdThrottleTimer.

StopIOTimer()

void XrdThrottleManager::StopIOTimer ( std::chrono::steady_clock::duration &  event_duration, uint16_t  uid  )

protected

Definition at line 744 of file XrdThrottleManager.cc.

{
   m_io_active_time += event_duration.count();
   auto old_active = m_io_active.fetch_sub(1, std::memory_order_acq_rel);
   m_waiter_info[uid].m_io_time += event_duration.count();
   if (old_active == static_cast<unsigned>(m_concurrency_limit))
   {
      // If we are below the concurrency limit threshold and have another waiter
      // for our user, then execute it immediately.  Otherwise, we will give
      // someone else a chance to run (as we have gotten more than our share recently).
      unsigned waiting_users = m_waiting_users;
      if (waiting_users == 0) waiting_users = 1;
      if (m_waiter_info[uid].m_concurrency < m_concurrency_limit / waiting_users)
      {
         std::unique_lock<std::mutex> lock(m_waiter_info[uid].m_mutex);
         if (m_waiter_info[uid].m_waiting > 0)
         {
            m_waiter_info[uid].NotifyOne(std::move(lock));
            return;
         }
      }
      NotifyOne();
   }
}

Friends And Related Function Documentation

XrdThrottleTimer

friend class XrdThrottleTimer

friend

Definition at line 56 of file XrdThrottleManager.hh.

Referenced by StartIOTimer().

---

The documentation for this class was generated from the following files:

• XrdThrottleManager.hh
• XrdThrottleManager.cc