$watch not triggering in directive after change in controller

Question

I have a directive which is basically just a countdown timer. The usage is fairly simple, start counting down whenever the startFlag is true, stop counting when it is false, and call the timeout function when time runs out:

app.directive('countdownTimer', [ '$interval', 
    function($interval) {
        var directive = {
            restrict: 'E',
            templateUrl: 'app/common/countdownTimer.html',
            scope: {
                timeAllotted: '=',
                startFlag: '=',
                timeoutCallback: '='
            },
            link: link
        };
        return directive;

        function link(scope, element, attrs) {
            var timeLeft;

            // Set watch on time allotted
            scope.$watch('timeAllotted', function(newValue) {
                timeLeft = newValue;
                scope.minutes = getMinutes(timeLeft);
                scope.seconds = getSeconds(timeLeft);
            });

            // Set watch on start flag
            scope.$watch('startFlag', function(newValue) {
                if(newValue) {
                    scope.timer = $interval(tickFunction, 1000);
                } else {
                    $interval.cancel(scope.timer);
                }
            });

            function getFormattedTimeValue(val) {
                if(val < 10) {
                    val = '0' +  val;
                }
                return val;
            }

            function getMinutes(secs) {
                return getFormattedTimeValue(parseInt(secs / 60));
            }

            function getSeconds(secs) {
                return getFormattedTimeValue(secs % 60);
            }

            function tickFunction(triggerTimeout) {
                timeLeft = timeLeft - 1;
                scope.minutes = getMinutes(timeLeft);
                scope.seconds = getSeconds(timeLeft);
                if(timeLeft <= 0) {
                    scope.timeoutCallback();
                    scope.startFlag = false;
                }
            }
        }
    }
]);

My issue is that the watch on timeAllotted above works, but only the first time. It is being set by a property of an object that is being loaded from a remote data source:

<countdown-timer
    data-time-allotted="vm.timeRemaining" 
    data-start-flag="vm.taskStarted" 
    data-timeout-callback="vm.taskTimeout">
</countdown-timer>

All of this works great. The directive updates its local timeLeft variable properly when the data from vm.timeRemaining changes (it is initialized at 0 and then updated when the other object becomes available).

However, I wish to also offer the option of extending the time once the timeout condition occurs. But when I update vm.timeRemaining after the timeout condition, the $watch on the directive is not triggered.

This is my current attempt to update the time remaining from my controller:

function extendTime() {
    // This should reset the clock back to it's starting time, 
    // but it doesn't change!
    vm.timeRemaining = vm.task.time_limit;

    vm.hasAccess = true;
    dismissModal();

    // Added timeout so I could try $scope.$apply().. didn't help
    $timeout(function() {

        $scope.$apply();

        // This confirms that vm.timeRemaining is updating here, but the 
        // $watch on timeAllotted isn't firing in the directive
        console.log('timeRemaining updated');
        console.log(vm.timeRemaining);

        startTask();
    }, 50);

}

For what it's worth, the task timeout isn't doing too much that's interesting:

function taskTimeout() {
    // Cancels timer
    pauseTask();
    // Synchronizes data with API
    syncTick();
    // Opens a modal
    vm.modalInstance = $modal.open({
        templateUrl: 'app/common/timeoutModal.html',
        scope: $scope
    });
}

Any idea why updating vm.timeRemaining in my controller's extendTime function isn't being recognized from my directive's $watch on it's timeAllotted bound variable?

Update

Below is the function that is retrieving a task from my datacontext (it is basically just a container of repositories). Once the promise is resolved, vm.timeRemaining is updated and, in this situation, is correctly $watched by the directive:

function getTask() {
    return datacontext.task.loadForLevel(level)
        .then(function(results) {
            vm.task = results[0];
            vm.timeRemaining = vm.task.time_limit;
        })
    ;
}