A/B testing examples

Example №1

Hypothesis

Our analytics data show that most of the people make a purchase during their first session. However, only 70% of users open our in-app store during the first session. We hypothesize that if we add a purchase screen to the tutorial then 100% of users will face it and the number of purchases during the first session will increase.

Test criteria

The criteria for test inclusion is starting the tutorial. The DTDAnalytics.tutorial(step: Int) event with step = -1 is the trigger.

Groups

Control group: a 10 step tutorial, no purchase screens

Group А: an 11 step tutorial. At step number 5, we will offer a special offer.

Implementation

iOS+macOS (Swift)

// Timer constants
struct Constants {
    static let waitGroupConst = 10.0
}

// Value keys
enum ValueKey: String {
  case showStore
}

class AppConfig {
    static func loadDefaults() {
        let appDefaults: [String: Any] = [
            ValueKey.showStore.rawValue: "false"
        ]
        // Set default values
        DTDRemoteConfig.defaults = appDefaults
    }
    
    static func bool(forKey key: ValueKey) -> Bool {
        return DTDRemoteConfig.config[key.rawValue].boolValue
    }
}

class AppLogic {
    var timer: Timer
    // Tutorial open event (e.g. by clicking the ‘start tutorial’ button)
    func startTutorial() {
        // Send a trigger event that indicates tutorial start
        DTDAnalytics.tutorial(step: -1)
    }
    
    func nextTutotrialStep(_ currentStep: Int) {
        DTDAnalytics.tutorial(step: currentStep)
        if AppConfig.bool(forKey: .showStore) && currentStep == 5 {
            // Offer purchasing of a special offer
        }
    }
}

class AppDelegate: UIResponder, UIApplicationDelegate {
    func application(_ application: UIApplication,
        willFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
        // Set the maximum time of waiting for an A/B test group
        DTDRemoteConfig.groupDefinitionWaiting = Constants.waitGroupConst
        // Set default values
        AppConfig.loadDefaults()
        // Initialize the SDK for working with A/B testing
        DTDAnalytics.initializeWithAbTest(applicationKey: "appKey",
                                          configuration: config,
                                          abConfigListener: self)
    }
}

extension AppLogic: DTDRemoteConfigListener {
    // Process the result of waiting for A/B test configuration
    func onReceived(result: DTDRemoteConfigReceiveResult) {
        // It is not used in current example
    }

    // Prepare the app UI for changing the remote configuration
    func onPrepareToChange() {
        // Use the main app thread because you are getting ready for working with the interface
        DispatchQueue.main.async { [weak self] in
            // Display the download progress indicator
            self?.showActivityIndicator()
            // Add a timer that will forcibly remove the download progress indicator
            self?.timer = Timer.scheduledTimer(withTimeInterval: Constants.waitGroupConst, repeats: false) { [weak self] _ in
                self?.hideActivityIndicator()
            }
        }
    }
    
    // Apply the values of the assigned group
    func onChanged(result: DTDRemoteConfigChangeResult, error: Error?) {
        defer {
            // Hide the download progress indicator
            DispatchQueue.main.async { [weak self] in
                self?.timer.invalidate();
                self?.hideActivityIndicator()
            }
        }

        switch result {
        case .success:
            // Apply new values
            DTDRemoteConfig.applyConfig()

        case .failure:
            // Error processing
            if let error = error {
                print(error.localizedDescription)
            }

        @unknown default:
            break
        }
    }
}

iOS+macOS (Objective-C)

Android (Kotlin)

Android (Java)

Unity

.Net + UWP

Unreal

Godot

Example №2

Hypothesis

Our analytics data show that N users installed the app more than half a year ago but still did not make a purchase. We hypothesize that if we offer a huge discount then we can get additional income.

Test criteria

The criteria for test inclusion is the app install date and the “payer” status.

Groups

Control group: current version with usual prices

Group А: it has a discount badge on the main page. After clicking on the badge, a purchase window pops up

Implementation

iOS+macOS (Swift)

 // Timer constants
struct Constants {
    static let waitConfigConst = 10.0
    static let waitGroupConst = 15.0
}

// Value keys
enum ValueKey: String {
  case maximumDiscount
}

class AppConfig {
    static func loadDefaults() {
        let appDefaults: [String: Any] = [
            ValueKey.maximumDiscount.rawValue: "false"
        ]
        // Set default values
        DTDRemoteConfig.defaults = appDefaults
    }
    
    static func bool(forKey key: ValueKey) -> Bool {
        return DTDRemoteConfig.config[key.rawValue].boolValue
    }
}

class AppLogic {
    override func viewDidLoad() {
        super.viewDidLoad()
        // Display the launch screen
        showLaunchScreen()
    }
    
    // Launching the main logic of the application
    func startAppForReal() {
        // Update app UI
        updateAppUI()
        // Hide launch screen
        hideLaunchScreen()
    }
    
    func updateAppUI() {
        if bool(forKey: .maximumDiscount) {
            // Display a discount badge clicking on which invokes a purchase window popup
        }
    }
}

class AppDelegate: UIResponder, UIApplicationDelegate {
    func application(_ application: UIApplication,
        willFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
        // Set the maximum time of waiting for the A/B test configuration
        DTDRemoteConfig.remoteConfigWaiting = Constants.waitConfigConst
        // Set the maximum time of waiting for an A/B test group
        DTDRemoteConfig.groupDefinitionWaiting = Constants.waitGroupConst
        // Set default values
        AppConfig.loadDefaults()
        // Initialize the SDK for working with A/B testing
        DTDAnalytics.initializeWithAbTest(applicationKey: "appKey",
                                          configuration: config,
                                          abConfigListener: self)
    }
}

extension AppLogic: DTDRemoteConfigListener {
    // Process the result of waiting for A/B test configuration
    func onReceived(result: DTDRemoteConfigReceiveResult) {
        // If the attempt fails, launch the main logic of the application
        if result == .failure {
            DispatchQueue.main.async { [weak self] in
                self?.startAppForReal()
            }
        }
    }

    // Prepare the app UI for changing the remote configuration
    func onPrepareToChange() {
        // It is not used in current example
    }
    
    // Apply the values of the assigned group
    func onChanged(result: DTDRemoteConfigChangeResult, error: Error?) {
        defer {
            // Launch the main logic of the application
            DispatchQueue.main.async { [weak self] in
              self?.startAppForReal()
            }
        }

        switch result {
        case .success:
            // Apply new values
            DTDRemoteConfig.applyConfig()

        case .failure:
            // Error processing
            if let error = error {
                print(error.localizedDescription)
            }

        @unknown default:
            break
        }
    }
}

Note

When receiving the configuration, the SDK always calls the onReceived(result: DTDRemoteConfigReceiveResult) method, and when enrolling in a test - the onPrepareToChange() and onChanged(result: DTDRemoteConfigChangeResult, error: Error?) method. However, you can take some additional precocious measures. You can add a timer as in the following example:

func showLaunchScreen() {
    // Add a timer that will forcibly launch the main logic of the application
    timer = Timer.scheduledTimer(withTimeInterval: waitConfigConst + waitGroupConst, 
                                          repeats: false) { [weak self] _ in
        self?.startAppForReal()
    }
    showActivityIndicator()
}

iOS+macOS (Objective-C)

Android (Kotlin)

Android (Java)

Unity

.NET + UWP

Unreal

Godot

Example №3

Hypothesis

Our analytics data show that only 60% of our users complete the tutorial. We hypothesize that if we make it easier (group A) or reduce the number of seps (group B), we will increase the percentage of tutorial completion.

Test criteria

New users who have not started the tutorial yet.

Groups

Control group: current version with usual difficulty and usual number of steps

Group А: low difficulty, usual number of steps

Group B: usual difficulty, few steps

Implementation

iOS+macOS (Swift)

// Timer constants
struct Constants {
    static let waitConfigConst = 10.0
}

// Value keys
enum ValueKey: String {
    case difficulty
    case stepCount
}

class AppConfig {
    static func loadDefaults() {
        func loadDefaultValues() {
        let appDefaults: [String: Any] = [
            ValueKey.difficulty.rawValue: 3,
            ValueKey.stepCount.rawValue: 10
        ]
        // Set default values
        DTDRemoteConfig.defaults = appDefaults
    }
    
    func integer(forKey key: ValueKey) -> Int {
        DTDRemoteConfig.config[key.rawValue].integerValue
    }
}

class AppLogic {
    // Tutorial open event (e.g. by clicking the ‘start tutorial’ button)
    func startTutorial() {
        // Display the download progress indicator
        showActivityIndicator()
        // Send a trigger event that indicates tutorial start
        DTDAnalytics.tutorial(step: -1)
    }

    // Initiate tutorial completion
    func realStartTutorial() {
        // Hide the download progress indicator
        hideActivityIndicator()
        // Initiate the tutorial by using the remote values (difficulty and stepCount)
        runTutorial(stepCount: integer(forKey: .stepCount), 
                    difficulty: integer(forKey: .difficulty))
    }
}

class AppDelegate: UIResponder, UIApplicationDelegate {
    func application(_ application: UIApplication,
        willFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
        // Set the maximum time of waiting for the A/B test configuration
        DTDRemoteConfig.remoteConfigWaiting = Constants.waitConfigConst
        // Set default values
        AppConfig.loadDefaults()
        // Initialize the SDK for working with A/B testing
        DTDAnalytics.initializeWithAbTest(applicationKey: "appKey",
                                          configuration: config,
                                          abConfigListener: self)
    }
}

extension AppLogic: DTDRemoteConfigListener {
    // Process the result of waiting for A/B test configuration
    func onReceived(result: DTDRemoteConfigReceiveResult) {
        // It is not used in current example
    }

    // Prepare the app UI for changing the remote configuration
    func onPrepareToChange() {
        // It is not used in current example
    }
    
    // Apply the values of the assigned group
    func onChanged(result: DTDRemoteConfigChangeResult, error: Error?) {
        defer {
            // Initiate tutorial completion
            DispatchQueue.main.async { [weak self] in
                self?.realStartTutorial()
            }
        }

        switch result {
        case .success:
            // Apply new values
            DTDRemoteConfig.applyConfig()

        case .failure:
            // Error processing
            if let error = error {
                print(error.localizedDescription)
            }

        @unknown default:
            break
        }
    }
}

iOS+macOS (Objective-C)

Android (Kotlin)

Android (Java)

Unity

.Net + UWP

Unreal

Godot