Image Resizer serves downscaled image

Question

I am using ImageResizer in an EPiServer 9 setup, where I've used the following plugin: https://nuget.episerver.com/en/OtherPages/Package/?packageId=ImageResizer.Plugins.EPiServerBlobReader along with ImageResizer.NET 4.0.5. I am implementing an AngularJS directive, where it updates the size if the size of the image. This is the implementation

app.directive('responsiveImage', function ($window, $timeout) {
    //From http://stackoverflow.com/a/6021027 with small modifications
    var setWidth = function (uri, value) {
        var parameterName = 'width';
        //REGEX for locating current parameter
        var re = new RegExp('([?&])' + parameterName + '=.*?(&|$)', 'i');
        if (uri.match(re)) {
            //Replaces the current w parameter with the new value
            return uri.replace(re, '$1' + parameterName + '=' + value + '$2');
        } else {
            //Identify which seperator to use
            var separator = uri.indexOf('?') !== -1 ? '&' : '?';
            //Sets the w parameter
            return uri + separator + parameterName + '=' + value;
        }
    };

    return {
        restrict: 'A',
        scope: {
            imageUrl: '@responsiveImage'
        },
        link: function(scope, element) {
            var maxWidth = 0;
            var promise;

            var resizeImage = function (initialCall) {
                $timeout.cancel(promise);

                promise = $timeout(function () {
                    var widthOfElement = element.width();

                    //Only fetch new image if the size is bigger, no need to downscale
                    if (widthOfElement > maxWidth) {
                        maxWidth = widthOfElement;

                        if (element.is('img')) {
                            element.attr('src', setWidth(scope.imageUrl, widthOfElement));
                            element.css('width', '');
                        } else { //Assume it is background image
                            element.css('background-image', "url('" + setWidth(scope.imageUrl, widthOfElement) + "')");
                        }
                    }
                    //If the first call, do it instantly but otherwise wait to see if the user is done resizing the window
                }, initialCall ? 0 : 50);
            };

            //Initial call
            resizeImage(true);

            angular.element($window).on('resize', resizeImage);
        }
    }
});

However when I start with a small window and resize it upwards, it stops serving the correct image and serves one of the low resolution images.

In the tests I've made, it got stuck at e.g. 1204px width. After it got stuck per se, then even if I browse to the image by typing it into the address bar and changing the width parameter, it serves a lower resolution image than asked. The original image is 2880px wide. I've tested in both Firefox and Chrome.

Is this a known issue in ImageResizer.NET when requesting too often that it returns an earlier image, or is there something wrong in my approach?