Firebird database UDF for encrypt/decrypt not freeing memory

Question

I'm testing a UDF external function in Firebird 3 database, I made a C++ DLL which performs a simple XOR to a given string using a given key.

This is the code:

#include <windows.h>
#include <iostream>
#include <string>
#include <stdio.h>
#include <vector>
#include <math.h>

#include "../FirebirdLib/src/include/ibase.h"
#include "ib_util.h"


using namespace std;

//------------------------------------------------------------------------------------
typedef void (__stdcall * FCallback)(const char * message);
FCallback g_messageCallback = 0;
FCallback g_errorCallback = 0;
//------------------------------------------------------------------------------------
#define ON_MESSAGE(mess) { if(g_messageCallback) g_messageCallback(mess); }
#define ON_ERROR(mess) { if(g_errorCallback) g_errorCallback(mess); }
//------------------------------------------------------------------------------------
extern "C" __declspec(dllexport) void RegisterCallbacks(FCallback messageCallback, FCallback errorCallback)
{
    g_messageCallback = messageCallback;
    g_errorCallback = errorCallback;
}
//------------------------------------------------------------------------------------
class EncryptionUDF
{
public:
    EncryptionUDF()
    {

        //ON_MESSAGE("--EncryptionUDF created--")
    }
    ~EncryptionUDF()
    {
        //ON_MESSAGE("--EncryptionUDF destroyed--")
    }

    char* XORCipher(const char* data, const char* key, int dataLen, int keyLen) {
        char* output = (char*)ib_util_malloc(2000 + 1L);
        output[dataLen] = '\0';
        for (int i = 0; i < dataLen; ++i) {
            if (data[i] != key[i % keyLen])
                output[i] = data[i] ^ key[i % keyLen];
            else 
                output[i] = data[i];
        }       

        return output;
    }

    char * Encrypt(const char * str, const char * key) {
        int dataLen = strlen(str);
        int keyLen = strlen(key);
        char* output = (char*)ib_util_malloc(2000 + 1L);
        output[dataLen] = '\0';

        try {
            if ((str == NULL) || (str[0] == '\0')) {
                return NULL;
            }
            else {
                try {
                    if ((key != NULL) && (key[0] == '\0')) {
                        strncpy(output, str, dataLen);
                    }
                    else if (key != NULL) {
                        output = XORCipher(str, key, dataLen, keyLen);
                    }
                    else strncpy(output, str, dataLen);
                }
                catch (...) { strncpy(output, str, dataLen); }

                return output;
            }
        }
        catch (...) { strncpy(output, str, dataLen); }

        return output;
    }

    char * Decrypt(const char * str, const char * key) {
        int dataLen = strlen(str);
        int keyLen = strlen(key);
        char* output = (char*)ib_util_malloc(2000 + 1L);
        output[dataLen] = '\0';

        try {
            if ((str == NULL) || (str[0] == '\0')) {
                return NULL;
            }
            else {
                try {
                    if ((key != NULL) && (key[0] == '\0')) {
                        strncpy(output, str, dataLen);
                    }
                    else if (key != NULL) {
                        output = XORCipher(str, key, dataLen, keyLen);
                    }
                    else strncpy(output, str, dataLen);
                }
                catch (...) { strncpy(output, str, dataLen); }

                return output;
            }
        }
        catch (...) { strncpy(output, str, dataLen); }

        return output;
    }
};
//------------------------------------------------------------------------------------
extern "C" __declspec(dllexport) char * EncryptUDF_DesEncrypt(const char *str, const char *key)
{
    try
    {
        EncryptionUDF self = EncryptionUDF();
        return self.Encrypt(str, key);
    }
    catch (std::exception & ex)
    {
        ON_ERROR(ex.what());
    }
    catch (...)
    {
        ON_ERROR("Unknown error");
    }
    return 0;
}
//------------------------------------------------------------------------------------
extern "C" __declspec(dllexport) char * EncryptUDF_DesDecrypt(const char *str, const char *key)
{
    try
    {
        EncryptionUDF self = EncryptionUDF();
        return self.Decrypt(str, key);
    }
    catch (std::exception & ex)
    {
        ON_ERROR(ex.what());
    }
    catch (...)
    {
        ON_ERROR("Unknown error");
    }
    return 0;
}
//------------------------------------------------------------------------------------
BOOL APIENTRY DllMain( HMODULE hModule,
                      DWORD  ul_reason_for_call,
                      LPVOID lpReserved
                      )
{
    return TRUE;
}
//------------------------------------------------------------------------------------

The UDF is defined in database as:

DECLARE EXTERNAL FUNCTION X_DECRYPT
  CSTRING(2000),
  CSTRING(64)
RETURNS CSTRING(2000) FREE_IT
ENTRY_POINT 'EncryptUDF_DesDecrypt' MODULE_NAME 'EncryptUDF';

DECLARE EXTERNAL FUNCTION X_ENCRYPT
  CSTRING(2000),
  CSTRING(64)
RETURNS CSTRING(2000) FREE_IT
ENTRY_POINT 'EncryptUDF_DesEncrypt' MODULE_NAME 'EncryptUDF';

When using this UDF in SQL select commands, the ram used by firebird server tends to increase continously. When using embedded the RAM goes up quickly, when in server mode, the RAM is increasing but slowly and somehow more controlled.

Please help to understand where the error is.

score 2 · Accepted Answer · answered Nov 08 '19 at 22:28

After some investigation, I decided to change the parts on code where the string is copied using:

strncpy(output, str, dataLen);

with:

strncpy_s(output, dataLen, str, dataLen);

and after this change, the memory has been in normal levels, either in embedded firebird or in server mode.

It seems that was a memory leak in releasing or managing those string copies.

Firebird database UDF for encrypt/decrypt not freeing memory

1 Answers1