com.enremmeta.rtb.caches

Class AdCache

java.lang.Object

com.enremmeta.rtb.caches.AdCache

All Implemented Interfaces:

Runnable

public class AdCache
extends Object
implements Runnable

This is a scheduled process that, on waking up, well, runs -- see that method for more info.

Author:

Gregory Golberg (grisha@alum.mit.edu)

Copyright © Enremmeta LLC 2015. All Rights Reserved.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private static class	AdCache.AdCollectionHolder
static class	AdCache.BudgetAllocationResult
class	AdCache.LoadResultHolder
class	AdCache.ValidationResultHolder

Field Summary

Fields

Modifier and Type	Field and Description
(package private) String	AD_CONFIG_SCRIPT_PACKAGE
(package private) String	AD_CONFIG_SCRIPT_SUBDIR
private AdCache.AdCollectionHolder	adCollectionHolder
private AdCache.AdCollectionHolder	adCollectionHolder2
private static FilenameFilter	adFilter
private DaoShortLivedMap<BidInFlightInfo>	bidInFlightInfoMap
private static FilenameFilter	bidPriceCalculatorFilter
private Map<String,BidPriceCalculator>	bpcIdToBpc
private com.lambdaworks.redis.RedisClient	client
private Map<String,ClientConfig>	clients
private AdCacheConfig	config
private DaoShortLivedMap<CacheableWebResponse>	cwrMap
private DaoShortLivedMap<String>	fcMap
private boolean	first
private static Map<String,Class<Lot49Plugin>>	GROOVY_CLASSES
private AtomicBoolean	isRunning
private DaoShortLivedMap<String>	nurlMap
private Map<String,byte[]>	ownerKeys
private static long	runNumber
private ScheduledFuture	scheduledSelf
private DbService	shortTermDb
private String	status
private Map<String,String>	tmpAdIdToBpcId Deprecated.
private int	ttlMinutes
private DaoCounters	winRateCounters
static Format	YYYYMMDD

Constructor Summary

Constructors

Constructor and Description
AdCache(AdCacheConfig config)

Method Summary

Modifier and Type	Method and Description
boolean	cancel() In order to run from AdminSvc.refreshAdCache(javax.ws.rs.container.AsyncResponse, String) or ...
private static String	debug(Ad ad, String s)
private static String	debug(org.apache.logging.log4j.Marker m, String s)
private static String	debug(String s)
private static String	debug(String adId, String s)
String	doRun(boolean scheduled) Ad cache refresh
private static String	error(Ad ad, String s)
private static String	error(Ad ad, String s, Throwable t)
private static String	error(String s)
private static String	error(String adId, String s, Throwable t)
private static String	error(String s, Throwable t)
private static String	formatMicroBudget(long value)
Ad	getAd(String id)
Ad[]	getAll()
Ad[]	getBid1()
Ad[]	getBid2()
private static BidderCalendar	getBidderCalendar()
DaoShortLivedMap<BidInFlightInfo>	getBidInFlightInfoMap()
List<Ad>	getBlackListedAds()
AdCacheConfig	getConfig()
DaoShortLivedMap<CacheableWebResponse>	getCwrMap()
DaoShortLivedMap<String>	getFcMap()
List<Ad>	getIneligibleAds()
Map<Ad,String>	getInvalidAds() Used for show list of invalid ads in the admin tool (see AdminSvc.index(javax.ws.rs.container.AsyncResponse, String))
private int	getNumberOfPeers(com.lambdaworks.redis.RedisConnection<String,String> redisCon)
DaoShortLivedMap<String>	getNurlMap()
byte[]	getOwnerKey(String owner)
private File	getReadingFile()
private com.lambdaworks.redis.RedisConnection<String,String>	getRedisConnection()
DbService	getShortTermDb()
String	getStatus()
Map<String,Long>	getTimeoutMap()
Map<String,String>	getUncompilableAds() Used for show list of uncompilable ads in the admin tool (see AdminSvc.index(javax.ws.rs.container.AsyncResponse, String))
List<Ad>	getZeroBudgetAds() Used for show list of ads with zero budget in the admin tool (see AdminSvc.index(javax.ws.rs.container.AsyncResponse, String))
private static String	info(Ad ad, String s)
private static String	info(org.apache.logging.log4j.Marker m, String s)
private static String	info(String s)
void	init()
private AdCache.ValidationResultHolder	isValidAd(String adId, com.lambdaworks.redis.RedisConnection<String,String> redisCon)
private AdCache.LoadResultHolder	load(groovy.lang.GroovyClassLoader loader, File f, Boolean forceUpdate) Load the groovy class.
private Ad	loadAd(File groovyPkgDir, groovy.lang.GroovyClassLoader loader, File f)
List<Ad>	loadAdsFromDir(String adsDir, String subDir, Boolean useSanboxClassloader, StringBuilder msg, com.lambdaworks.redis.RedisConnection<String,String> redisCon)
private BidPriceCalculator	loadBidPriceCalculator(File groovyPkgDir, groovy.lang.GroovyClassLoader loader, File f)
private Boolean	loadTag(groovy.lang.GroovyClassLoader loader, File f)
private Boolean	loadTagsByAd(File groovyPkgDir, groovy.lang.GroovyClassLoader loader, String adFileName, String adId)
private static String	logPrefix()
String	refresh(com.lambdaworks.redis.RedisConnection<String,String> redisCon)
private String	refreshOwnerKeys(com.lambdaworks.redis.RedisConnection<String,String> redisCon)
private AdCache.AdCollectionHolder	replaceHolders()
private String	rescheduleSelf()
private AdCache.BudgetAllocationResult	reserveMoney(com.lambdaworks.redis.RedisConnection<String,String> redisCon, Ad ad) Reserve money from the main budget cache for a given ad.
void	run() Same as doRun(true)
String	setBudget(String adId, long amount) Only for use in Admin tools.
private String	setStatus(String s)
private void	tmpPopulateAdIdToBpcId() Deprecated.
private static String	warn(Ad ad, String s)
private static String	warn(Ad ad, String s, Throwable t)
private static String	warn(String s)
private static String	warn(String adId, String s)
private static String	warn(String adId, String s, Throwable t)
private static String	warn(String s, Throwable t)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

runNumber

private static long runNumber

YYYYMMDD

public static final Format YYYYMMDD

AD_CONFIG_SCRIPT_PACKAGE

final String AD_CONFIG_SCRIPT_PACKAGE

See Also:

Constant Field Values

AD_CONFIG_SCRIPT_SUBDIR

final String AD_CONFIG_SCRIPT_SUBDIR

config

private final AdCacheConfig config

clients

private final Map<String,ClientConfig> clients

ttlMinutes

private final int ttlMinutes

status

private String status

client

private final com.lambdaworks.redis.RedisClient client

adCollectionHolder

private volatile AdCache.AdCollectionHolder adCollectionHolder

adCollectionHolder2

private AdCache.AdCollectionHolder adCollectionHolder2

GROOVY_CLASSES

private static final Map<String,Class<Lot49Plugin>> GROOVY_CLASSES

bpcIdToBpc

private final Map<String,BidPriceCalculator> bpcIdToBpc

cwrMap

private DaoShortLivedMap<CacheableWebResponse> cwrMap

winRateCounters

private DaoCounters winRateCounters

nurlMap

private DaoShortLivedMap<String> nurlMap

fcMap

private DaoShortLivedMap<String> fcMap

bidInFlightInfoMap

private DaoShortLivedMap<BidInFlightInfo> bidInFlightInfoMap

shortTermDb

private DbService shortTermDb

adFilter

private static final FilenameFilter adFilter

bidPriceCalculatorFilter

private static final FilenameFilter bidPriceCalculatorFilter

first

private boolean first

ownerKeys

private Map<String,byte[]> ownerKeys

tmpAdIdToBpcId

@Deprecated
private final Map<String,String> tmpAdIdToBpcId

Deprecated.

scheduledSelf

private ScheduledFuture scheduledSelf

isRunning

private AtomicBoolean isRunning

Constructor Detail

AdCache

public AdCache(AdCacheConfig config)
        throws Lot49Exception

Throws:

Lot49Exception

Method Detail

getFcMap

public DaoShortLivedMap<String> getFcMap()

getConfig

public AdCacheConfig getConfig()

setBudget

public String setBudget(String adId,
                        long amount)
                 throws Lot49Exception

Only for use in Admin tools.

Throws:

Lot49Exception

getStatus

public String getStatus()

getAd

public Ad getAd(String id)

getZeroBudgetAds

public List<Ad> getZeroBudgetAds()

Used for show list of ads with zero budget in the admin tool (see AdminSvc.index(javax.ws.rs.container.AsyncResponse, String))

getIneligibleAds

public List<Ad> getIneligibleAds()

getBlackListedAds

public List<Ad> getBlackListedAds()

getInvalidAds

public Map<Ad,String> getInvalidAds()

Used for show list of invalid ads in the admin tool (see AdminSvc.index(javax.ws.rs.container.AsyncResponse, String))

getUncompilableAds

public Map<String,String> getUncompilableAds()

Used for show list of uncompilable ads in the admin tool (see AdminSvc.index(javax.ws.rs.container.AsyncResponse, String))

getTimeoutMap

public Map<String,Long> getTimeoutMap()

load

private AdCache.LoadResultHolder load(groovy.lang.GroovyClassLoader loader,
                                      File f,
                                      Boolean forceUpdate)
                               throws org.codehaus.groovy.control.CompilationFailedException,
                                      IOException,
                                      InstantiationException,
                                      IllegalAccessException

Load the groovy class. The contract is such that this logs an error and returns null if not successful.

Throws:

org.codehaus.groovy.control.CompilationFailedException

IOException

InstantiationException

IllegalAccessException

getBidderCalendar

private static BidderCalendar getBidderCalendar()

loadTagsByAd

private Boolean loadTagsByAd(File groovyPkgDir,
                             groovy.lang.GroovyClassLoader loader,
                             String adFileName,
                             String adId)

loadTag

private Boolean loadTag(groovy.lang.GroovyClassLoader loader,
                        File f)

loadAd

private Ad loadAd(File groovyPkgDir,
                  groovy.lang.GroovyClassLoader loader,
                  File f)

loadBidPriceCalculator

private BidPriceCalculator loadBidPriceCalculator(File groovyPkgDir,
                                                  groovy.lang.GroovyClassLoader loader,
                                                  File f)
                                           throws Lot49Exception

Throws:

Lot49Exception

init

public void init()
          throws Lot49Exception

Throws:

Lot49Exception

getShortTermDb

public DbService getShortTermDb()

getCwrMap

public DaoShortLivedMap<CacheableWebResponse> getCwrMap()

getNurlMap

public DaoShortLivedMap<String> getNurlMap()

getBidInFlightInfoMap

public DaoShortLivedMap<BidInFlightInfo> getBidInFlightInfoMap()

getBid1

public Ad[] getBid1()

getBid2

public Ad[] getBid2()

getAll

public Ad[] getAll()

formatMicroBudget

private static final String formatMicroBudget(long value)

reserveMoney

private AdCache.BudgetAllocationResult reserveMoney(com.lambdaworks.redis.RedisConnection<String,String> redisCon,
                                                    Ad ad)
                                             throws Lot49Exception

Reserve money from the main budget cache for a given ad. The algorithm is as follows:

Parameters:

redisCon - Connection to the pacing cache.

ad -

Returns:

Throws:

Lot49Exception

getReadingFile

private File getReadingFile()

getOwnerKey

public byte[] getOwnerKey(String owner)

refreshOwnerKeys

private String refreshOwnerKeys(com.lambdaworks.redis.RedisConnection<String,String> redisCon)

getNumberOfPeers

private int getNumberOfPeers(com.lambdaworks.redis.RedisConnection<String,String> redisCon)
                      throws Lot49Exception

Throws:

Lot49Exception

isValidAd

private AdCache.ValidationResultHolder isValidAd(String adId,
                                                 com.lambdaworks.redis.RedisConnection<String,String> redisCon)

tmpPopulateAdIdToBpcId

@Deprecated
private void tmpPopulateAdIdToBpcId()
                                         throws Lot49Exception

Deprecated.

Throws:

Lot49Exception

loadAdsFromDir

public List<Ad> loadAdsFromDir(String adsDir,
                               String subDir,
                               Boolean useSanboxClassloader,
                               StringBuilder msg,
                               com.lambdaworks.redis.RedisConnection<String,String> redisCon)
                        throws Lot49Exception

Throws:

Lot49Exception

refresh

public String refresh(com.lambdaworks.redis.RedisConnection<String,String> redisCon)

replaceHolders

private final AdCache.AdCollectionHolder replaceHolders()

logPrefix

private static final String logPrefix()

error

private static final String error(String s)

error

private static final String error(String s,
                                  Throwable t)

warn

private static final String warn(String s)

warn

private static final String warn(String s,
                                 Throwable t)

info

private static final String info(String s)

info

private static final String info(org.apache.logging.log4j.Marker m,
                                 String s)

debug

private static final String debug(String s)

debug

private static final String debug(org.apache.logging.log4j.Marker m,
                                  String s)

error

private static final String error(Ad ad,
                                  String s)

error

private static final String error(Ad ad,
                                  String s,
                                  Throwable t)

error

private static final String error(String adId,
                                  String s,
                                  Throwable t)

warn

private static final String warn(Ad ad,
                                 String s)

warn

private static final String warn(String adId,
                                 String s)

warn

private static final String warn(Ad ad,
                                 String s,
                                 Throwable t)

warn

private static final String warn(String adId,
                                 String s,
                                 Throwable t)

info

private static final String info(Ad ad,
                                 String s)

debug

private static final String debug(Ad ad,
                                  String s)

debug

private static final String debug(String adId,
                                  String s)

cancel

public boolean cancel()

In order to run from AdminSvc.refreshAdCache(javax.ws.rs.container.AsyncResponse, String) or ...

rescheduleSelf

private String rescheduleSelf()

getRedisConnection

private final com.lambdaworks.redis.RedisConnection<String,String> getRedisConnection()
                                                                               throws Lot49Exception

Throws:

Lot49Exception

setStatus

private String setStatus(String s)

doRun

public String doRun(boolean scheduled)

Ad cache refresh

This method is run every minute. It can also be forced explicitly to run, usually in the debug phase. We do guarantee, however, that only one instance of this method is running. It goes through a number of housekeeping tasks.

Ad refresh

This is one of the main tasks. It does the following:

Refresh ads

This only runs every ttl minutes. It goes through the following steps:

Parameters:

scheduled - whether the run was scheduled or explicit (say, via invocation from AdminSvc.refreshAdCache(javax.ws.rs.container.AsyncResponse, String)).

Returns:

log messages to print (in case of explicit invocation).

run

public void run()

Same as doRun(true)

Specified by:

run in interface Runnable

See Also:

doRun(boolean)