A/B testing examples

Example №1

Hypothesis

Our analytics data show that most of the people make a purchase during their first session. However, only 70% of users open our in-app store during the first session. We hypothesize that if we add a purchase screen to the tutorial then 100% of users will face it and the number of purchases during the first session will increase.

Test criteria

The criteria for test inclusion is starting the tutorial. The DTDAnalytics.tutorial(step: Int) event with step = -1 is the trigger.

Groups

Control group: a 10 step tutorial, no purchase screens

Group А: an 11 step tutorial. At step number 5, we will offer a special offer.

Implementation

// Timer constants
struct Constants {
    static let waitGroupConst = 10.0
}

// Value keys
enum ValueKey: String {
  case showStore
}

class AppConfig {
    static func loadDefaults() {
        let appDefaults: [String: Any] = [
            ValueKey.showStore.rawValue: "false"
        ]
        // Set default values
        DTDRemoteConfig.defaults = appDefaults
    }
    
    static func bool(forKey key: ValueKey) -> Bool {
        return DTDRemoteConfig.config[key.rawValue].boolValue
    }
}

class AppLogic {
    var timer: Timer
    // Tutorial open event (e.g. by clicking the ‘start tutorial’ button)
    func startTutorial() {
        // Send a trigger event that indicates tutorial start
        DTDAnalytics.tutorial(step: -1)
    }
    
    func nextTutotrialStep(_ currentStep: Int) {
        DTDAnalytics.tutorial(step: currentStep)
        if AppConfig.bool(forKey: .showStore) && currentStep == 5 {
            // Offer purchasing of a special offer
        }
    }
}

class AppDelegate: UIResponder, UIApplicationDelegate {
    func application(_ application: UIApplication,
        willFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
        // Set the maximum time of waiting for an A/B test group
        DTDRemoteConfig.groupDefinitionWaiting = Constants.waitGroupConst
        // Set default values
        AppConfig.loadDefaults()
        // Initialize the SDK for working with A/B testing
        DTDAnalytics.initializeWithAbTest(applicationKey: "appKey",
                                          configuration: config,
                                          abConfigListener: self)
    }
}

extension AppLogic: DTDRemoteConfigListener {
    // Process the result of waiting for A/B test configuration
    func onReceived(result: DTDRemoteConfigReceiveResult) {
        // It is not used in current example
    }

    // Prepare the app UI for changing the remote configuration
    func onPrepareToChange() {
        // Use the main app thread because you are getting ready for working with the interface
        DispatchQueue.main.async { [weak self] in
            // Display the download progress indicator
            self?.showActivityIndicator()
            // Add a timer that will forcibly remove the download progress indicator
            self?.timer = Timer.scheduledTimer(withTimeInterval: Constants.waitGroupConst, repeats: false) { [weak self] _ in
                self?.hideActivityIndicator()
            }
        }
    }
    
    // Apply the values of the assigned group
    func onChanged(result: DTDRemoteConfigChangeResult, error: Error?) {
        defer {
            // Hide the download progress indicator
            DispatchQueue.main.async { [weak self] in
                self?.timer.invalidate();
                self?.hideActivityIndicator()
            }
        }

        switch result {
        case .success:
            // Apply new values
            DTDRemoteConfig.applyConfig()

        case .failure:
            // Error processing
            if let error = error {
                print(error.localizedDescription)
            }

        @unknown default:
            break
        }
    }
}

Example №2

Hypothesis

Our analytics data show that N users installed the app more than half a year ago but still did not make a purchase. We hypothesize that if we offer a huge discount then we can get additional income.

Test criteria

The criteria for test inclusion is the app install date and the “payer” status.

Groups

Control group: current version with usual prices

Group А: it has a discount badge on the main page. After clicking on the badge, a purchase window pops up

Implementation

 // Timer constants
struct Constants {
    static let waitConfigConst = 10.0
    static let waitGroupConst = 15.0
}

// Value keys
enum ValueKey: String {
  case maximumDiscount
}

class AppConfig {
    static func loadDefaults() {
        let appDefaults: [String: Any] = [
            ValueKey.maximumDiscount.rawValue: "false"
        ]
        // Set default values
        DTDRemoteConfig.defaults = appDefaults
    }
    
    static func bool(forKey key: ValueKey) -> Bool {
        return DTDRemoteConfig.config[key.rawValue].boolValue
    }
}

class AppLogic {
    override func viewDidLoad() {
        super.viewDidLoad()
        // Display the launch screen
        showLaunchScreen()
    }
    
    // Launching the main logic of the application
    func startAppForReal() {
        // Update app UI
        updateAppUI()
        // Hide launch screen
        hideLaunchScreen()
    }
    
    func updateAppUI() {
        if bool(forKey: .maximumDiscount) {
            // Display a discount badge clicking on which invokes a purchase window popup
        }
    }
}

class AppDelegate: UIResponder, UIApplicationDelegate {
    func application(_ application: UIApplication,
        willFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
        // Set the maximum time of waiting for the A/B test configuration
        DTDRemoteConfig.remoteConfigWaiting = Constants.waitConfigConst
        // Set the maximum time of waiting for an A/B test group
        DTDRemoteConfig.groupDefinitionWaiting = Constants.waitGroupConst
        // Set default values
        AppConfig.loadDefaults()
        // Initialize the SDK for working with A/B testing
        DTDAnalytics.initializeWithAbTest(applicationKey: "appKey",
                                          configuration: config,
                                          abConfigListener: self)
    }
}

extension AppLogic: DTDRemoteConfigListener {
    // Process the result of waiting for A/B test configuration
    func onReceived(result: DTDRemoteConfigReceiveResult) {
        // If the attempt fails, launch the main logic of the application
        if result == .failure {
            DispatchQueue.main.async { [weak self] in
                self?.startAppForReal()
            }
        }
    }

    // Prepare the app UI for changing the remote configuration
    func onPrepareToChange() {
        // It is not used in current example
    }
    
    // Apply the values of the assigned group
    func onChanged(result: DTDRemoteConfigChangeResult, error: Error?) {
        defer {
            // Launch the main logic of the application
            DispatchQueue.main.async { [weak self] in
              self?.startAppForReal()
            }
        }

        switch result {
        case .success:
            // Apply new values
            DTDRemoteConfig.applyConfig()

        case .failure:
            // Error processing
            if let error = error {
                print(error.localizedDescription)
            }

        @unknown default:
            break
        }
    }
}

Note

When receiving the configuration, the SDK always calls the onReceived(result: DTDRemoteConfigReceiveResult) method, and when enrolling in a test - the onPrepareToChange() and onChanged(result: DTDRemoteConfigChangeResult, error: Error?) method. However, you can take some additional precocious measures. You can add a timer as in the following example:

func showLaunchScreen() {
    // Add a timer that will forcibly launch the main logic of the application
    timer = Timer.scheduledTimer(withTimeInterval: waitConfigConst + waitGroupConst, 
                                          repeats: false) { [weak self] _ in
        self?.startAppForReal()
    }
    showActivityIndicator()
}